From 88cf6ffce3c01f4f1c50250ef46c0d7bf23f41c7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 16:27:00 -0400 Subject: [PATCH 001/227] Rename logging->stdout to logging->stdout_ musl doesn't like this identifier --- src/libutil/logging.hh | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/hash.cc | 4 ++-- src/nix/ls.cc | 4 ++-- src/nix/show-config.cc | 2 +- src/nix/why-depends.cc | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index b1583eced..46deb89f7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -87,7 +87,7 @@ public: virtual void writeToStdout(std::string_view s); template - inline void stdout(const std::string & fs, const Args & ... args) + inline void stdout_(const std::string & fs, const Args & ... args) { boost::format f(fs); formatHelper(f, args...); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d6de16e..745c24748 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -58,7 +58,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand store->addToStore(info, source); } - logger->stdout("%s", store->printStorePath(info.path)); + logger->stdout_("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 26e98ac2a..53ec8c920 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -65,7 +65,7 @@ struct CmdEval : MixJSON, InstallableCommand printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - logger->stdout("%s", *v); + logger->stdout_("%s", *v); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..cdc8bf767 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -69,7 +69,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->stdout(h.to_string(base, base == SRI)); + logger->stdout_(h.to_string(base, base == SRI)); } } }; @@ -103,7 +103,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout(Hash(s, ht).to_string(base, base == SRI)); + logger->stdout_(Hash(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d2157f2d4..59922a8de 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); - logger->stdout(line); + logger->stdout_(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - logger->stdout(relPath); + logger->stdout_(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 4fd8886de..a97dc42f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -25,7 +25,7 @@ struct CmdShowConfig : Command, MixJSON std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - logger->stdout("%s = %s", s.first, s.second.value); + logger->stdout_("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 167c974ee..5e4d5fdcf 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -152,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - logger->stdout("%s%s%s%s" ANSI_NORMAL, + logger->stdout_("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From 07dae2ff7727b915a1b687cdec9aca894d7c2f72 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:19:11 -0400 Subject: [PATCH 002/227] Setup static building of nix --- release-common.nix | 37 ++++++++------ release.nix | 117 ++++++++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/release-common.nix b/release-common.nix index 4316c3c23..2cf9c233e 100644 --- a/release-common.nix +++ b/release-common.nix @@ -1,4 +1,4 @@ -{ pkgs }: +{ pkgs, enableStatic }: with pkgs; @@ -30,35 +30,42 @@ rec { }); configureFlags = - lib.optionals stdenv.isLinux [ + lib.optionals (!enableStatic && stdenv.isLinux) [ "--with-sandbox-shell=${sh}/bin/busybox" ]; + nativeBuildDeps = + [ + buildPackages.bison + buildPackages.flex + buildPackages.libxml2 + buildPackages.libxslt + buildPackages.docbook5 + buildPackages.docbook_xsl_ns + buildPackages.autoreconfHook + buildPackages.pkgconfig + + # Tests + buildPackages.git + buildPackages.mercurial + buildPackages.ipfs + ]; + buildDeps = - [ bison - flex - libxml2 - libxslt - docbook5 - docbook_xsl_ns + [ autoreconfHook autoconf-archive - autoreconfHook curl bzip2 xz brotli zlib editline - openssl pkgconfig sqlite + openssl sqlite libarchive boost nlohmann_json - - # Tests - git - mercurial gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + ++ lib.optional (!enableStatic && (stdenv.isLinux || stdenv.isDarwin)) ((aws-sdk-cpp.override { apis = ["s3" "transfer"]; customMemoryManagement = false; diff --git a/release.nix b/release.nix index fbf9e4721..d2785be13 100644 --- a/release.nix +++ b/release.nix @@ -12,63 +12,72 @@ let builtins.readFile ./.version + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); + buildFun = pkgs: enableStatic: + with pkgs; with import ./release-common.nix { inherit pkgs enableStatic; }; + stdenv.mkDerivation { + name = "nix-${version}"; + + src = nix; + + outputs = [ "out" "dev" "doc" ]; + + buildInputs = buildDeps; + + nativeBuildInputs = nativeBuildDeps; + + propagatedBuildInputs = propagatedDeps; + + preConfigure = + lib.optionalString (!enableStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + + (cd perl; autoreconf --install --force --verbose) + ''; + + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + dontUpdateAutotoolsGnuConfigScripts = true; + + enableParallelBuilding = true; + + makeFlags = [ "profiledir=$(out)/etc/profile.d" "PRECOMPILE_HEADERS=0" ]; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ''; + + doCheck = true; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + + separateDebugInfo = !enableStatic; + + stripAllList = ["bin"]; + }; + + jobs = rec { + + build-static = pkgs.lib.genAttrs systems (system: + buildFun (import nixpkgs { inherit system; }).pkgsStatic true); + + build = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - stdenv.mkDerivation { - name = "nix-${version}"; - - src = nix; - - outputs = [ "out" "dev" "doc" ]; - - buildInputs = buildDeps; - - propagatedBuildInputs = propagatedDeps; - - preConfigure = - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - - (cd perl; autoreconf --install --force --verbose) - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ''; - - doCheck = true; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - - separateDebugInfo = true; - }); + buildFun (import nixpkgs { inherit system; }) false); perlBindings = pkgs.lib.genAttrs systems (system: From 70719a9dd8c3d91e1d6a83d4ec9a48023cddaecf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:20:06 -0400 Subject: [PATCH 003/227] Add -lz to end of linking this is needed for static linking to work properly --- mk/programs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/programs.mk b/mk/programs.mk index 3fa9685c3..a96ff56af 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz $(1)_INSTALL_DIR ?= $$(bindir) From da77331cb740ad7d5f39dcf6d64025610ec40555 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:20:29 -0400 Subject: [PATCH 004/227] Remove lazy lookup in getHome this seems to break in Musl/Static with: terminate called after throwing an instance of 'std::bad_function_call' what(): bad_function_call --- src/libutil/lazy.hh | 48 --------------------------------------------- src/libutil/util.cc | 6 +++--- 2 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 src/libutil/lazy.hh diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh deleted file mode 100644 index d073e486c..000000000 --- a/src/libutil/lazy.hh +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include - -namespace nix { - -/* A helper class for lazily-initialized variables. - - Lazy var([]() { return value; }); - - declares a variable of type T that is initialized to 'value' (in a - thread-safe way) on first use, that is, when var() is first - called. If the initialiser code throws an exception, then all - subsequent calls to var() will rethrow that exception. */ -template -class Lazy -{ - - typedef std::function Init; - - Init init; - - std::once_flag done; - - T value; - - std::exception_ptr ex; - -public: - - Lazy(Init init) : init(init) - { } - - const T & operator () () - { - std::call_once(done, [&]() { - try { - value = init(); - } catch (...) { - ex = std::current_exception(); - } - }); - if (ex) std::rethrow_exception(ex); - return value; - } -}; - -} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1268b146a..ebb1383f3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,3 @@ -#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -511,7 +510,8 @@ std::string getUserName() } -static Lazy getHome2([]() { +static Path getHome2() +{ auto homeDir = getEnv("HOME"); if (!homeDir) { std::vector buf(16384); @@ -523,7 +523,7 @@ static Lazy getHome2([]() { homeDir = pw->pw_dir; } return *homeDir; -}); +}; Path getHome() { return getHome2(); } From 289558dffbccb47191e79629e955009b10b9888e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:39:35 -0400 Subject: [PATCH 005/227] Add unordered_set to globals.cc header --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index bee94cbd8..fa8799314 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -8,7 +8,7 @@ #include #include #include - +#include namespace nix { From 78fadaf863536d23d45b2c480139a9f77d579f9e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:44:57 -0400 Subject: [PATCH 006/227] fix release.nix eval --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index d2785be13..863db900b 100644 --- a/release.nix +++ b/release.nix @@ -191,7 +191,7 @@ let coverage = with pkgs; - with import ./release-common.nix { inherit pkgs; }; + with import ./release-common.nix { inherit pkgs; enableStatic = false; }; releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; From ded65899538f6a4628e711abbbdf27ea47772742 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 18:04:16 -0400 Subject: [PATCH 007/227] Fixup coverage build --- release.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/release.nix b/release.nix index 863db900b..60cbca17b 100644 --- a/release.nix +++ b/release.nix @@ -200,6 +200,7 @@ let enableParallelBuilding = true; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps ++ propagatedDeps; dontInstall = false; From 24da034bc3ae8514ae19dadfecf6038452a5290a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 21:21:27 +0000 Subject: [PATCH 008/227] Add possibly missing `` include --- src/libutil/types.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0..2170e4c93 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include From 696bb134c1c5882cf258e3c8a480b40239cb1a9a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 21:36:09 +0000 Subject: [PATCH 009/227] Fix shell.nix --- shell.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell.nix b/shell.nix index 17aaa05ed..1addc06be 100644 --- a/shell.nix +++ b/shell.nix @@ -1,8 +1,8 @@ -{ useClang ? false }: +{ useClang ? false, enableStatic ? false }: with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {}; -with import ./release-common.nix { inherit pkgs; }; +with import ./release-common.nix { inherit pkgs enableStatic; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; From baaab2aab58aa3c47517d4ba9121a29a7ad73078 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jun 2020 14:53:40 +0000 Subject: [PATCH 010/227] Add `nativeBuildInputs` to shell.nix --- shell.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell.nix b/shell.nix index 1addc06be..75bb6ac1b 100644 --- a/shell.nix +++ b/shell.nix @@ -7,6 +7,8 @@ with import ./release-common.nix { inherit pkgs enableStatic; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ propagatedDeps ++ perlDeps; inherit configureFlags; From 13ef7a07b9be1dff894e8156a117ee3248241874 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:49:45 -0500 Subject: [PATCH 011/227] Fix build --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index bf2066157..40d779355 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,7 @@ # Tests buildPackages.git buildPackages.mercurial + buildPackages.jq ]; buildDeps = From 3537670fefab6c65b4b87837112d64931dcda4cf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:49:52 -0500 Subject: [PATCH 012/227] Only enable static on linux --- flake.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 40d779355..20c5089ee 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,8 @@ officialRelease = false; - systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; + systems = linuxSystems ++ [ "x86_64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); @@ -209,7 +210,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - build-static = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix-static); + build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -423,13 +424,14 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - build-static = self.hydraJobs.build-static.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { + build-static = self.hydraJobs.build-static.${system}; }); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; - + } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { From e12bcabdcbddc228d7af157bb3c2090e324c59a7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 02:30:12 +0000 Subject: [PATCH 013/227] Remove duplicate buildInputs --- flake.nix | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index 20c5089ee..843f9a85f 100644 --- a/flake.nix +++ b/flake.nix @@ -69,6 +69,7 @@ buildPackages.libxslt buildPackages.docbook5 buildPackages.docbook_xsl_ns + buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkgconfig @@ -79,18 +80,9 @@ ]; buildDeps = - [ bison - flex - libxml2 - libxslt - docbook5 - docbook_xsl_ns - autoconf-archive - autoreconfHook - - curl + [ curl bzip2 xz brotli zlib editline - openssl pkgconfig sqlite + openssl sqlite libarchive boost (if lib.versionAtLeast lib.version "20.03pre" @@ -178,14 +170,17 @@ src = self; + nativeBuildInputs = + [ buildPackages.autoconf-archive + buildPackages.autoreconfHook + buildPackages.pkgconfig + ]; + buildInputs = - [ autoconf-archive - autoreconfHook - nix + [ nix curl bzip2 xz - pkgconfig pkgs.perl boost ] From ec14465a001387f8972c1b8332293d4fbce5ec97 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 02:43:56 +0000 Subject: [PATCH 014/227] Separate lowdown lib and bin to be more precise --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 82eea55e2..9c79c0bbf 100644 --- a/flake.nix +++ b/flake.nix @@ -66,7 +66,7 @@ [ buildPackages.bison buildPackages.flex - buildPackages.lowdown + (lib.getBin buildPackages.lowdown) buildPackages.mdbook buildPackages.autoconf-archive buildPackages.autoreconfHook @@ -208,7 +208,7 @@ src = lowdown-src; - outputs = [ "out" "dev" ]; + outputs = [ "out" "bin" "dev" ]; nativeBuildInputs = [ which ]; @@ -216,7 +216,7 @@ '' ./configure \ PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "out"}/bin + BINDIR=${placeholder "bin"}/bin ''; }; From cfe791a638a3fdf53a2608f885c407bafc238094 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Sep 2020 11:30:04 -0400 Subject: [PATCH 015/227] stdout_ -> cout Better to get creative than just sprinkle arbitrary underscores. --- src/libutil/logging.hh | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 36 ++++++++++++++++++------------------ src/nix/hash.cc | 4 ++-- src/nix/ls.cc | 4 ++-- src/nix/profile.cc | 2 +- src/nix/registry.cc | 2 +- src/nix/search.cc | 6 +++--- src/nix/show-config.cc | 4 ++-- src/nix/why-depends.cc | 2 +- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 77b92fb51..e3fe613e8 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -100,7 +100,7 @@ public: virtual void writeToStdout(std::string_view s); template - inline void stdout_(const std::string & fs, const Args & ... args) + inline void cout(const std::string & fs, const Args & ... args) { boost::format f(fs); formatHelper(f, args...); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index df55d1bc4..04ab664b3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -83,7 +83,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand store->addToStore(info, source); } - logger->stdout_("%s", store->printStorePath(info.path)); + logger->cout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 9689f2bdb..754ffc911 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableCommand printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - logger->stdout_("%s", *v); + logger->cout("%s", *v); } } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 64fc896d9..90ffaad7c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -62,17 +62,17 @@ public: static void printFlakeInfo(const Store & store, const Flake & flake) { - logger->stdout_("Resolved URL: %s", flake.resolvedRef.to_string()); - logger->stdout_("Locked URL: %s", flake.lockedRef.to_string()); + logger->cout("Resolved URL: %s", flake.resolvedRef.to_string()); + logger->cout("Locked URL: %s", flake.lockedRef.to_string()); if (flake.description) - logger->stdout_("Description: %s", *flake.description); - logger->stdout_("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); + logger->cout("Description: %s", *flake.description); + logger->cout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); if (auto rev = flake.lockedRef.input.getRev()) - logger->stdout_("Revision: %s", rev->to_string(Base16, false)); + logger->cout("Revision: %s", rev->to_string(Base16, false)); if (auto revCount = flake.lockedRef.input.getRevCount()) - logger->stdout_("Revisions: %s", *revCount); + logger->cout("Revisions: %s", *revCount); if (auto lastModified = flake.lockedRef.input.getLastModified()) - logger->stdout_("Last modified: %s", + logger->cout("Last modified: %s", std::put_time(std::localtime(&*lastModified), "%F %T")); } @@ -140,7 +140,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON if (json) { auto json = flakeToJson(*store, flake); - logger->stdout_("%s", json.dump()); + logger->cout("%s", json.dump()); } else printFlakeInfo(*store, flake); } @@ -158,9 +158,9 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON auto flake = lockFlake(); if (json) - logger->stdout_("%s", flake.lockFile.toJson()); + logger->cout("%s", flake.lockFile.toJson()); else { - logger->stdout_("%s", flake.flake.lockedRef); + logger->cout("%s", flake.flake.lockedRef); std::unordered_set> visited; @@ -172,7 +172,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); @@ -180,7 +180,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); } else if (auto follows = std::get_if<1>(&input.second)) { - logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), input.first, printInputPath(*follows)); } @@ -811,7 +811,7 @@ struct CmdFlakeShow : FlakeCommand try { auto recurse = [&]() { - logger->stdout_("%s", headerPrefix); + logger->cout("%s", headerPrefix); auto attrs = visitor.getAttrs(); for (const auto & [i, attr] : enumerate(attrs)) { bool last = i + 1 == attrs.size(); @@ -837,7 +837,7 @@ struct CmdFlakeShow : FlakeCommand } */ - logger->stdout_("%s: %s '%s'", + logger->cout("%s: %s '%s'", headerPrefix, attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : @@ -885,7 +885,7 @@ struct CmdFlakeShow : FlakeCommand if (attrPath.size() == 1) recurse(); else if (!showLegacy) - logger->stdout_("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); + logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); else { if (visitor.isDerivation()) showDerivation(); @@ -902,7 +902,7 @@ struct CmdFlakeShow : FlakeCommand auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") throw EvalError("not an app definition"); - logger->stdout_("%s: app", headerPrefix); + logger->cout("%s: app", headerPrefix); } else if ( @@ -910,11 +910,11 @@ struct CmdFlakeShow : FlakeCommand (attrPath.size() == 2 && attrPath[0] == "templates")) { auto description = visitor.getAttr("description")->getString(); - logger->stdout_("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); } else { - logger->stdout_("%s: %s", + logger->cout("%s: %s", headerPrefix, attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 52e41e50c..945d8e990 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -73,7 +73,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->stdout_(h.to_string(base, base == SRI)); + logger->cout(h.to_string(base, base == SRI)); } } }; @@ -107,7 +107,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout_(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index b1cf92692..a9664e7c3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); - logger->stdout_(line); + logger->cout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - logger->stdout_(relPath); + logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f97df4d9e..def7db03b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -389,7 +389,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - logger->stdout_("%d %s %s %s", i, + logger->cout("%d %s %s %s", i, element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index afa3503b8..941785e55 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -26,7 +26,7 @@ struct CmdRegistryList : StoreCommand for (auto & registry : registries) { for (auto & entry : registry->entries) { // FIXME: format nicely - logger->stdout_("%s %s %s", + logger->cout("%s %s %s", registry->type == Registry::Flag ? "flags " : registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : diff --git a/src/nix/search.cc b/src/nix/search.cc index 88815efdb..2f7eb23bb 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -147,13 +147,13 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("description", description); } else { auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); - if (results > 1) logger->stdout_(""); - logger->stdout_( + if (results > 1) logger->cout(""); + logger->cout( "* %s%s", wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") - logger->stdout_( + logger->cout( " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); } } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 01a49f107..328cd2ff2 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -20,12 +20,12 @@ struct CmdShowConfig : Command, MixJSON { if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). - logger->stdout_("%s", globalConfig.toJSON().dump()); + logger->cout("%s", globalConfig.toJSON().dump()); } else { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - logger->stdout_("%s = %s", s.first, s.second.value); + logger->cout("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index cbfc9b948..f49d19ab2 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -156,7 +156,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - logger->stdout_("%s%s%s%s" ANSI_NORMAL, + logger->cout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From bd5328814fe8055b3f832a087afcf3ef11b06372 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Sat, 26 Sep 2020 14:32:58 -0700 Subject: [PATCH 016/227] Add some internal documentation for flake support objects. --- src/libexpr/flake/flake.hh | 29 +++++++++++++++++++++++++---- src/libexpr/flake/flakeref.hh | 15 +++++++++++++++ src/libfetchers/fetchers.hh | 16 ++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 69c779af8..40476a137 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,20 +17,41 @@ struct FlakeInput; typedef std::map FlakeInputs; +// FlakeInput is the flake-level parsed form of the "input" entries in +// the flake file. +// +// A FlakeInput is normally constructed by initially +// first constructing a FlakeRef (a fetcher, the fetcher-specific +// representation of the input specification, and the fetched local +// store path result) and then creating this FlakeInput to hold that +// FlakeRef, along with anything that might override that FlakeRef +// (like command-line overrides or "follows" specifications). +// +// A FlakeInput is also sometimes constructed directly from a FlakeRef +// instead of starting at the flake-file input specification +// (e.g. overrides, follows, and implicit inputs). +// +// A FlakeInput will usually have one of either "ref" or "follows" +// set. If not otherwise specified, a "ref" will be generated to a +// 'type="indirect"' flake, which is treated as simply the name of a +// flake to be resolved in the registry. + struct FlakeInput { std::optional ref; - bool isFlake = true; + bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path std::optional follows; bool absolute = false; // whether 'follows' is relative to the flake root FlakeInputs overrides; }; +// The Flake structure is the main internal representation of a flake.nix file. + struct Flake { - FlakeRef originalRef; - FlakeRef resolvedRef; - FlakeRef lockedRef; + FlakeRef originalRef; // the original flake specification (by the user) + FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake + FlakeRef lockedRef; // the specific local store result of invoking the fetcher std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index f4eb825a6..ac68cde0e 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,10 +12,25 @@ class Store; typedef std::string FlakeId; +// The FlakeRef represents a local nix store reference to a flake +// input for a Flake (it may be helpful to think of this object by the +// alternate name of "InputRefForFlake"). It is constructed by +// starting with an input description (usually the attrs or a url from +// the flake file), locating a fetcher for that input, and then +// capturing the Input object that fetcher generates (usually via +// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). +// +// The actual fetch not have been performed yet (i.e. a FlakeRef may +// be lazy), but the fetcher can be invoked at any time via the +// FlakeRef to ensure the store is populated with this input. + struct FlakeRef { + // fetcher-specific representation of the input, sufficient to + // perform the fetch operation. fetchers::Input input; + // sub-path within the fetched input that represents this input Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 89b1e6e7d..f361edf17 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,6 +21,13 @@ struct Tree struct InputScheme; +// The Input object is generated by a specific fetcher, based on the +// user-supplied input attribute in the flake.nix file, and contians +// the information that the specific fetcher needs to perform the +// actual fetch. The Input object is most commonly created via the +// "fromURL()" or "fromAttrs()" static functions which are provided the +// url or attrset specified in the flake file. + struct Input { friend struct InputScheme; @@ -82,6 +89,15 @@ public: std::optional getLastModified() const; }; + +// The InputScheme represents a type of fetcher. Each fetcher +// registers with nix at startup time. When processing an input for a +// flake, each scheme is given an opportunity to "recognize" that +// input from the url or attributes in the flake file's specification +// and return an Input object to represent the input if it is +// recognized. The Input object contains the information the fetcher +// needs to actually perform the "fetch()" when called. + struct InputScheme { virtual std::optional inputFromURL(const ParsedURL & url) = 0; From bcb3da3b6b510bd24f2bc973b39bf43d92fad7ce Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 08:58:14 -0700 Subject: [PATCH 017/227] Fix spelling error. --- src/libfetchers/fetchers.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index f361edf17..6aa8a3422 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -22,7 +22,7 @@ struct Tree struct InputScheme; // The Input object is generated by a specific fetcher, based on the -// user-supplied input attribute in the flake.nix file, and contians +// user-supplied input attribute in the flake.nix file, and contains // the information that the specific fetcher needs to perform the // actual fetch. The Input object is most commonly created via the // "fromURL()" or "fromAttrs()" static functions which are provided the From 5ae164b7cf2dd7ca1846f349b57131913aa7cf55 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:23:05 -0700 Subject: [PATCH 018/227] Update description of FlakeRef, incorporating suggestion. --- src/libexpr/flake/flakeref.hh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index ac68cde0e..2596dd18e 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,12 +12,19 @@ class Store; typedef std::string FlakeId; -// The FlakeRef represents a local nix store reference to a flake -// input for a Flake (it may be helpful to think of this object by the -// alternate name of "InputRefForFlake"). It is constructed by -// starting with an input description (usually the attrs or a url from -// the flake file), locating a fetcher for that input, and then -// capturing the Input object that fetcher generates (usually via +// A flake reference specifies how to fetch a flake or raw source +// (e.g. from a Git repository). It is created from a URL-like syntax +// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ +// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local +// path. +// +// Each flake will have a number of FlakeRef objects: one for each +// input to the flake. +// +// The normal method of constructing a FlakeRef is by starting with an +// input description (usually the attrs or a url from the flake file), +// locating a fetcher for that input, and then capturing the Input +// object that fetcher generates (usually via // FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). // // The actual fetch not have been performed yet (i.e. a FlakeRef may From 128c98ab0961ba234774508663f591758d3a2178 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:34:23 -0700 Subject: [PATCH 019/227] Clarification in the description of the FlakeInput. --- src/libexpr/flake/flake.hh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 40476a137..44cf636b5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,15 +17,16 @@ struct FlakeInput; typedef std::map FlakeInputs; -// FlakeInput is the flake-level parsed form of the "input" entries in +// FlakeInput is the 'Flake'-level parsed form of the "input" entries in // the flake file. // -// A FlakeInput is normally constructed by initially -// first constructing a FlakeRef (a fetcher, the fetcher-specific -// representation of the input specification, and the fetched local -// store path result) and then creating this FlakeInput to hold that -// FlakeRef, along with anything that might override that FlakeRef -// (like command-line overrides or "follows" specifications). +// A FlakeInput is normally constructed by the 'parseFlakeInput' +// function which parses the input specification in the '.flake' file +// to create a 'FlakeRef' (a fetcher, the fetcher-specific +// representation of the input specification, and possibly the fetched +// local store path result) and then creating this FlakeInput to hold +// that FlakeRef, along with anything that might override that +// FlakeRef (like command-line overrides or "follows" specifications). // // A FlakeInput is also sometimes constructed directly from a FlakeRef // instead of starting at the flake-file input specification From 887be7b6f233822f03d156dafa156276d59162fd Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:37:26 -0700 Subject: [PATCH 020/227] Switch comment format from '// ...' to '/* ... */' for consistency. --- src/libexpr/flake/flake.hh | 39 +++++++++++++++---------------- src/libexpr/flake/flakeref.hh | 43 ++++++++++++++++++----------------- src/libfetchers/fetchers.hh | 28 ++++++++++++----------- 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 44cf636b5..cf62c7741 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,25 +17,26 @@ struct FlakeInput; typedef std::map FlakeInputs; -// FlakeInput is the 'Flake'-level parsed form of the "input" entries in -// the flake file. -// -// A FlakeInput is normally constructed by the 'parseFlakeInput' -// function which parses the input specification in the '.flake' file -// to create a 'FlakeRef' (a fetcher, the fetcher-specific -// representation of the input specification, and possibly the fetched -// local store path result) and then creating this FlakeInput to hold -// that FlakeRef, along with anything that might override that -// FlakeRef (like command-line overrides or "follows" specifications). -// -// A FlakeInput is also sometimes constructed directly from a FlakeRef -// instead of starting at the flake-file input specification -// (e.g. overrides, follows, and implicit inputs). -// -// A FlakeInput will usually have one of either "ref" or "follows" -// set. If not otherwise specified, a "ref" will be generated to a -// 'type="indirect"' flake, which is treated as simply the name of a -// flake to be resolved in the registry. +/* FlakeInput is the 'Flake'-level parsed form of the "input" entries + * in the flake file. + * + * A FlakeInput is normally constructed by the 'parseFlakeInput' + * function which parses the input specification in the '.flake' file + * to create a 'FlakeRef' (a fetcher, the fetcher-specific + * representation of the input specification, and possibly the fetched + * local store path result) and then creating this FlakeInput to hold + * that FlakeRef, along with anything that might override that + * FlakeRef (like command-line overrides or "follows" specifications). + * + * A FlakeInput is also sometimes constructed directly from a FlakeRef + * instead of starting at the flake-file input specification + * (e.g. overrides, follows, and implicit inputs). + * + * A FlakeInput will usually have one of either "ref" or "follows" + * set. If not otherwise specified, a "ref" will be generated to a + * 'type="indirect"' flake, which is treated as simply the name of a + * flake to be resolved in the registry. + */ struct FlakeInput { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 2596dd18e..0292eb210 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,32 +12,33 @@ class Store; typedef std::string FlakeId; -// A flake reference specifies how to fetch a flake or raw source -// (e.g. from a Git repository). It is created from a URL-like syntax -// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ -// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local -// path. -// -// Each flake will have a number of FlakeRef objects: one for each -// input to the flake. -// -// The normal method of constructing a FlakeRef is by starting with an -// input description (usually the attrs or a url from the flake file), -// locating a fetcher for that input, and then capturing the Input -// object that fetcher generates (usually via -// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). -// -// The actual fetch not have been performed yet (i.e. a FlakeRef may -// be lazy), but the fetcher can be invoked at any time via the -// FlakeRef to ensure the store is populated with this input. +/* A flake reference specifies how to fetch a flake or raw source + * (e.g. from a Git repository). It is created from a URL-like syntax + * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ + * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local + * path. + * + * Each flake will have a number of FlakeRef objects: one for each + * input to the flake. + * + * The normal method of constructing a FlakeRef is by starting with an + * input description (usually the attrs or a url from the flake file), + * locating a fetcher for that input, and then capturing the Input + * object that fetcher generates (usually via + * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). + * + * The actual fetch not have been performed yet (i.e. a FlakeRef may + * be lazy), but the fetcher can be invoked at any time via the + * FlakeRef to ensure the store is populated with this input. + */ struct FlakeRef { - // fetcher-specific representation of the input, sufficient to - // perform the fetch operation. + /* fetcher-specific representation of the input, sufficient to + perform the fetch operation. */ fetchers::Input input; - // sub-path within the fetched input that represents this input + /* sub-path within the fetched input that represents this input */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6aa8a3422..84c954899 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,12 +21,13 @@ struct Tree struct InputScheme; -// The Input object is generated by a specific fetcher, based on the -// user-supplied input attribute in the flake.nix file, and contains -// the information that the specific fetcher needs to perform the -// actual fetch. The Input object is most commonly created via the -// "fromURL()" or "fromAttrs()" static functions which are provided the -// url or attrset specified in the flake file. +/* The Input object is generated by a specific fetcher, based on the + * user-supplied input attribute in the flake.nix file, and contains + * the information that the specific fetcher needs to perform the + * actual fetch. The Input object is most commonly created via the + * "fromURL()" or "fromAttrs()" static functions which are provided + * the url or attrset specified in the flake file. + */ struct Input { @@ -90,13 +91,14 @@ public: }; -// The InputScheme represents a type of fetcher. Each fetcher -// registers with nix at startup time. When processing an input for a -// flake, each scheme is given an opportunity to "recognize" that -// input from the url or attributes in the flake file's specification -// and return an Input object to represent the input if it is -// recognized. The Input object contains the information the fetcher -// needs to actually perform the "fetch()" when called. +/* The InputScheme represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an input for a + * flake, each scheme is given an opportunity to "recognize" that + * input from the url or attributes in the flake file's specification + * and return an Input object to represent the input if it is + * recognized. The Input object contains the information the fetcher + * needs to actually perform the "fetch()" when called. + */ struct InputScheme { From aef44cbaa9aaa6fa40eb34c411cedd858e16101e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:38:46 +0000 Subject: [PATCH 021/227] Split out `commonChildInit` --- src/libstore/build.cc | 29 ----------------------------- src/libutil/util.cc | 29 +++++++++++++++++++++++++++++ src/libutil/util.hh | 2 ++ 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 05a9ef088..5757e5c3f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -456,35 +456,6 @@ void Goal::trace(const FormatOrString & fs) ////////////////////////////////////////////////////////////////////// -/* Common initialisation performed in child processes. */ -static void commonChildInit(Pipe & logPipe) -{ - restoreSignals(); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9804e9a51..53342b5cb 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1660,4 +1660,33 @@ string showBytes(uint64_t bytes) } +void commonChildInit(Pipe & logPipe) +{ + const static string pathNullDevice = "/dev/null"; + restoreSignals(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 129d59a97..cafe93702 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -536,6 +536,8 @@ typedef std::function PathFilter; extern PathFilter defaultPathFilter; +/* Common initialisation performed in child processes. */ +void commonChildInit(Pipe & logPipe); /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); From 428536fd75a8318a5b9a5274f071d4beb39d58dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:39:08 +0000 Subject: [PATCH 022/227] Prepare for build/* files --- src/libstore/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d266c8efe..212a725af 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil From fc72cb07601dde8266ff2afd57f30381a66bb4c3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:16:29 +0000 Subject: [PATCH 023/227] Rename to hand-hold git (build.hh) --- src/libstore/{build.cc => build.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build.hh} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build.hh similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build.hh From a4f0fecb032ae074f4d2395e2740b393ee740849 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:17:24 +0000 Subject: [PATCH 024/227] Trim build.hh --- src/libstore/build.hh | 4914 +---------------------------------------- 1 file changed, 22 insertions(+), 4892 deletions(-) diff --git a/src/libstore/build.hh b/src/libstore/build.hh index 5757e5c3f..8027c61b1 100644 --- a/src/libstore/build.hh +++ b/src/libstore/build.hh @@ -1,24 +1,11 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" +#pragma once +#include "machines.hh" +#include "parsed-derivations.hh" +#include "lock.hh" +#include "local-store.hh" + +#include #include #include #include @@ -28,60 +15,12 @@ #include #include #include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - namespace nix { using std::map; -static string pathNullDevice = "/dev/null"; - - /* Forward definition. */ class Worker; struct HookInstance; @@ -184,14 +123,6 @@ struct Goal : public std::enable_shared_from_this void amDone(ExitCode result, std::optional ex = {}); }; - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - typedef std::chrono::time_point steady_time_point; @@ -372,332 +303,6 @@ public: } }; - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -1072,3648 +677,6 @@ private: StorePathSet exportReferences(const StorePathSet & storePaths); }; - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - class SubstitutionGoal : public Goal { friend class Worker; @@ -4792,863 +755,30 @@ public: StorePath getStorePath() { return storePath; } }; - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) +struct HookInstance { - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} + /* Pipes for talking to the build hook. */ + Pipe toHook; + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + /* The process ID of the hook. */ + Pid pid; -void SubstitutionGoal::work() -{ - (this->*state)(); -} + FdSink sink; + std::map activities; -void SubstitutionGoal::init() -{ - trace("init"); + HookInstance(); - worker.store.addTempRoot(storePath); + ~HookInstance(); +}; - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} +void addToWeakGoals(WeakGoals & goals, GoalPtr p); } From 9629290eda0828f842a9a5d1965c237d921091f8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:18:50 +0000 Subject: [PATCH 025/227] Rename to hand-hold git (build/derivation-goal.cc) --- src/libstore/{build.cc => build/derivation-goal.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/derivation-goal.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/derivation-goal.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/derivation-goal.cc From 3bab1c5bb0a56f850a7bc1bacc9f974b108cf601 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:19:10 +0000 Subject: [PATCH 026/227] Trim build/derivation-goal.cc --- src/libstore/build/derivation-goal.cc | 1960 +------------------------ 1 file changed, 15 insertions(+), 1945 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5757e5c3f..bb8921759 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,53 +1,31 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" +#include "build.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "filetransfer.hh" +#include "references.hh" #include "finally.hh" -#include "compression.hh" +#include "util.hh" +#include "archive.hh" #include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" +#include "compression.hh" #include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include #include -#include -#include -#include #include #include -#include #include -#include -#include -#include +#include #include -#include +#include +#include +#include +#include -#include -#include +#if HAVE_STATVFS +#include +#endif /* Includes required for chroot support. */ #if __linux__ @@ -67,395 +45,13 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif -#if HAVE_STATVFS -#include -#endif +#include +#include #include - namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, @@ -488,594 +84,8 @@ void handleDiffHook( } } -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - const Path DerivationGoal::homeDir = "/homeless-shelter"; - DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -4711,944 +3721,4 @@ void DerivationGoal::done(BuildResult::Status status, std::optional ex) } -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From 184bfc301e19ce9b11ace6ae0bd5a753da4e2931 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:19:55 +0000 Subject: [PATCH 027/227] Rename to hand-hold git (build/goal.cc) --- src/libstore/{build.cc => build/goal.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/goal.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/goal.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/goal.cc From 819fe848ac7a1fe88af5b648266a356c86224586 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:20:28 +0000 Subject: [PATCH 028/227] Trim build/goal.cc --- src/libstore/build/goal.cc | 5568 +----------------------------------- 1 file changed, 1 insertion(+), 5567 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 5757e5c3f..404a430f7 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,189 +1,7 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - +#include "build.hh" namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { string s1 = a->key(); @@ -192,190 +10,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - void addToWeakGoals(WeakGoals & goals, GoalPtr p) { // FIXME: necessary? @@ -451,5204 +85,4 @@ void Goal::trace(const FormatOrString & fs) debug("%1%: %2%", name, fs.s); } - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From f0b8987299285c398a6db2ca32f3007a441c3a90 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:21:30 +0000 Subject: [PATCH 029/227] Rename to hand-hold git (build/hook-instance.cc) --- src/libstore/{build.cc => build/hook-instance.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/hook-instance.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/hook-instance.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/hook-instance.cc From 159054f7305c2161f78074570232199e160dad57 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:21:54 +0000 Subject: [PATCH 030/227] Trim build/hook-instance.cc --- src/libstore/build/hook-instance.cc | 5585 +-------------------------- 1 file changed, 1 insertion(+), 5584 deletions(-) diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 5757e5c3f..a33d5e22b 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,633 +1,7 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - +#include "build.hh" namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - HookInstance::HookInstance() { debug("starting build hook '%s'", settings.buildHook); @@ -694,4961 +68,4 @@ HookInstance::~HookInstance() } } - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From dc5225cde5aa1cd240483caf08f4efdaab11ddae Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:25:04 +0000 Subject: [PATCH 031/227] Rename to hand-hold git (build/local-store-build.cc) --- src/libstore/{build.cc => build/local-store-build.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/local-store-build.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/local-store-build.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/local-store-build.cc From 4bdff7d1b0040dee7fdd51b5338f79290375c490 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:25:25 +0000 Subject: [PATCH 032/227] Trim build/local-store-build.cc --- src/libstore/build/local-store-build.cc | 5533 +---------------------- 1 file changed, 1 insertion(+), 5532 deletions(-) diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc index 5757e5c3f..407979b1e 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/local-store-build.cc @@ -1,5537 +1,7 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - +#include "build.hh" namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - static void primeCache(Store & store, const std::vector & paths) { StorePathSet willBuild, willSubstitute, unknown; @@ -5650,5 +120,4 @@ void LocalStore::repairPath(const StorePath & path) } } - } From 3633b3572b66c4fdbbff1f578687bf5926da969b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:25:58 +0000 Subject: [PATCH 033/227] Rename to hand-hold git (build/substitution-goal.cc) --- src/libstore/{build.cc => build/substitution-goal.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/substitution-goal.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/substitution-goal.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/substitution-goal.cc From d24ffe0eb146cbc48b9006415c795ad38efd6db0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:26:10 +0000 Subject: [PATCH 034/227] Trim build/substitution-goal.cc --- src/libstore/build/substitution-goal.cc | 5363 +---------------------- 1 file changed, 2 insertions(+), 5361 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 5757e5c3f..dd2ce94a6 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4798 +1,9 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" +#include "build.hh" #include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - +#include "finally.hh" namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) @@ -5081,574 +292,4 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From 904e315dae7a937465b762a0c0e8ccb4ccd27dc0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:26:54 +0000 Subject: [PATCH 035/227] Rename to hand-hold git (build/worker.cc) --- src/libstore/{build.cc => build/worker.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => build/worker.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/build/worker.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/build/worker.cc From eed53ed87aed55d8a9ad7d0edae6ef37f14b9988 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:28:49 +0000 Subject: [PATCH 036/227] Trim build/worker.cc --- src/libstore/build/worker.cc | 5205 +--------------------------------- 1 file changed, 1 insertion(+), 5204 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 5757e5c3f..2fc9f6982 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,5089 +1,9 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" +#include "build.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - - namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - Worker::Worker(LocalStore & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -5528,127 +448,4 @@ void Worker::markContentsGood(const StorePath & path) pathContentsGoodCache.insert_or_assign(path, true); } - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From d0004bfcab0741e76c65f28e46b9c06a7979d49c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:33:18 +0000 Subject: [PATCH 037/227] Rename to hand-hold git (lock.hh) --- src/libstore/{build.cc => lock.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => lock.hh} (100%) diff --git a/src/libstore/build.cc b/src/libstore/lock.hh similarity index 100% rename from src/libstore/build.cc rename to src/libstore/lock.hh From dbc588651c2e2e8952852e390c1d09f61a275f94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:34:36 +0000 Subject: [PATCH 038/227] Trim lock.hh --- src/libstore/lock.hh | 5625 +----------------------------------------- 1 file changed, 4 insertions(+), 5621 deletions(-) diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 5757e5c3f..8fbb67ddc 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,496 +1,11 @@ -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" -#include "local-store.hh" +#pragma once + +#include "sync.hh" +#include "types.hh" #include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - class UserLock { private: @@ -519,5136 +34,4 @@ public: }; - -UserLock::UserLock() -{ - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - killUser(uid); -} - - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From e0be04129b058680b3b3c64098cf3a0aa9d3901e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:35:06 +0000 Subject: [PATCH 039/227] Rename to hand-hold git (lock.cc) --- src/libstore/{build.cc => lock.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.cc => lock.cc} (100%) diff --git a/src/libstore/build.cc b/src/libstore/lock.cc similarity index 100% rename from src/libstore/build.cc rename to src/libstore/lock.cc From bcb67e1ed8ac3d8eea8c9b7c2f69c9ea2ab8c12b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:35:19 +0000 Subject: [PATCH 040/227] Trim lock.cc --- src/libstore/lock.cc | 5571 +----------------------------------------- 1 file changed, 5 insertions(+), 5566 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 5757e5c3f..f1356fdca 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,525 +1,15 @@ -#include "references.hh" -#include "pathlocks.hh" +#include "lock.hh" #include "globals.hh" -#include "local-store.hh" -#include "util.hh" -#include "archive.hh" -#include "affinity.hh" -#include "builtins.hh" -#include "builtins/buildenv.hh" -#include "filetransfer.hh" -#include "finally.hh" -#include "compression.hh" -#include "json.hh" -#include "nar-info.hh" -#include "parsed-derivations.hh" -#include "machines.hh" -#include "daemon.hh" -#include "worker-protocol.hh" -#include "topo-sort.hh" -#include "callback.hh" +#include "pathlocks.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include #include +#include -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if HAVE_STATVFS -#include -#endif - -#include - +#include +#include namespace nix { -using std::map; - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); - return s1 < s2; -} - - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - - -////////////////////////////////////////////////////////////////////// - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const FormatOrString & fs) -{ - debug("%1%: %2%", name, fs.s); -} - - - -////////////////////////////////////////////////////////////////////// - - -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); - logError(ei); - } - } -} - -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - UserLock::UserLock() { assert(settings.buildUsersGroup != ""); @@ -600,5055 +90,4 @@ void UserLock::kill() killUser(uid); } - -////////////////////////////////////////////////////////////////////// - - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -HookInstance::HookInstance() -{ - debug("starting build hook '%s'", settings.buildHook); - - /* Create a pipe to get the output of the child. */ - fromHook.create(); - - /* Create the communication pipes. */ - toHook.create(); - - /* Create a pipe to get the output of the builder. */ - builderOut.create(); - - /* Fork the hook. */ - pid = startProcess([&]() { - - commonChildInit(fromHook); - - if (chdir("/") == -1) throw SysError("changing into /"); - - /* Dup the communication pipes. */ - if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide.get(), 4) == -1) - throw SysError("dupping builder's stdout/stderr"); - - /* Hack: pass the read side of that fd to allow build-remote - to read SSH error messages. */ - if (dup2(builderOut.readSide.get(), 5) == -1) - throw SysError("dupping builder's stdout/stderr"); - - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); - }); - - pid.setSeparatePG(true); - fromHook.writeSide = -1; - toHook.readSide = -1; - - sink = FdSink(toHook.writeSide.get()); - std::map settings; - globalConfig.getSettings(settings); - for (auto & setting : settings) - sink << 1 << setting.first << setting.second.value; - sink << 0; -} - - -HookInstance::~HookInstance() -{ - try { - toHook.writeSide = -1; - if (pid != -1) pid.kill(); - } catch (...) { - ignoreException(); - } -} - - -////////////////////////////////////////////////////////////////////// - - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(BasicDerivation(drv)); - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} - - -DerivationGoal::~DerivationGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - -void DerivationGoal::killChild() -{ - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makeSubstitutionGoal(i)); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makeSubstitutionGoal(i, Repair)); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); -} - - -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - -void replaceValidPath(const Path & storePath, const Path & tmpPath) -{ - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); - - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); -} - - -MakeError(NotDeterministic, BuildError); - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); - - sandboxMountNamespace = -1; - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; - - /* Close the log file. */ - closeLogFile(); - - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - deleteTmpDir(true); - done(BuildResult::Built); - return; - } - - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - machineName = readLine(hook->fromHook.readSide.get()); - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); - } - worker_proto::write(worker.store, hook->sink, missingPaths); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} - - -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - - dirsInChroot.clear(); - - for (auto i : dirs) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - pid_t tmp; - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - - -void DerivationGoal::registerOutputs() -{ - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressed derivations this isn't the case. - */ - if (hook) { - bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) - allValid = false; - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); - continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif - - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - logError({ - .name = "Output determinism error", - .hint = hint - }); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); - } - worker.store.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); -} - - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - initialOutputs.insert_or_assign(i.first, info); - } -} - - -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - - -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) - { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &SubstitutionGoal::referencesValid; -} - - -void SubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &SubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - amDone(ecSuccess); -} - - -void SubstitutionGoal::handleChildOutput(int fd, const string & data) -{ -} - - -void SubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - -////////////////////////////////////////////////////////////////////// - - -Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) - , store(store) -{ - /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; -} - - -Worker::~Worker() -{ - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); - - assert(expectedSubstitutions == 0); - assert(expectedDownloadSize == 0); - assert(expectedNarSize == 0); -} - - -std::shared_ptr Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const StringSet & wantedOutputs, - std::function()> mkDrvGoal) -{ - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { - goal = mkDrvGoal(); - abstract_goal_weak = goal; - wakeUp(goal); - } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); - }); -} - - -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) -{ - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); - }); -} - - -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) -{ - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::removeGoal(GoalPtr goal) -{ - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - - waitingForAnyGoal.clear(); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - addToWeakGoals(awake, goal); -} - - -unsigned Worker::getNrLocalBuilds() -{ - return nrLocalBuilds; -} - - -void Worker::childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - child.respectTimeouts = respectTimeouts; - children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; -} - - -void Worker::childTerminated(Goal * goal, bool wakeSleepers) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - } - - children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) - wakeUp(goal); /* we can do it right away */ - else - addToWeakGoals(wantingToBuild, goal); -} - - -void Worker::waitForAnyGoal(GoalPtr goal) -{ - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); -} - - -void Worker::run(const Goals & _topGoals) -{ - for (auto & i : _topGoals) topGoals.insert(i); - - debug("entered goal loop"); - - while (1) { - - checkInterrupt(); - - store.autoGC(false); - - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - goal->work(); - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - - if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - - } - assert(!awake.empty()); - } - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); -} - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (!i.respectTimeouts) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } - - if (goal->exitCode == Goal::ecBusy && - 0 != settings.maxSilentTime && - j->respectTimeouts && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime)); - } - - else if (goal->exitCode == Goal::ecBusy && - 0 != settings.buildTimeout && - j->respectTimeouts && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - goal->timedOut(Error( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout)); - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } -} - - -unsigned int Worker::exitStatus() -{ - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - -bool Worker::pathContentsGood(const StorePath & path) -{ - auto i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printInfo("checking path '%s'...", store.printStorePath(path)); - auto info = store.queryPathInfo(path); - bool res; - if (!pathExists(store.printStorePath(path))) - res = false; - else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; - } - pathContentsGoodCache.insert_or_assign(path, res); - if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); - return res; -} - - -void Worker::markContentsGood(const StorePath & path) -{ - pathContentsGoodCache.insert_or_assign(path, true); -} - - -////////////////////////////////////////////////////////////////////// - - -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - primeCache(*this, drvPaths); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - primeCache(*this, {{path}}); - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - - } From 5c74a6147b4b81dc5b173f190f02f6681ec4b0fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 17:07:14 +0000 Subject: [PATCH 041/227] Properly type the derivation and substitution goal maps As a bonus, Worker::removeGoal is less inefficient. --- src/libstore/build.hh | 11 +++++------ src/libstore/build/worker.cc | 33 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/build.hh b/src/libstore/build.hh index 8027c61b1..894feab5b 100644 --- a/src/libstore/build.hh +++ b/src/libstore/build.hh @@ -28,7 +28,6 @@ struct HookInstance; /* A pointer to a goal. */ struct Goal; -class DerivationGoal; typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -140,6 +139,8 @@ struct Child steady_time_point timeStarted; }; +class DerivationGoal; +class SubstitutionGoal; /* The worker class. */ class Worker @@ -167,8 +168,8 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; + std::map> derivationGoals; + std::map> substitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -242,7 +243,7 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -305,8 +306,6 @@ public: typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -class SubstitutionGoal; - /* Unless we are repairing, we don't both to test validity and just assume it, so the choices are `Absent` or `Valid`. */ enum struct PathStatus { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2fc9f6982..47403580e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -39,16 +39,13 @@ std::shared_ptr Worker::makeDerivationGoalCommon( const StringSet & wantedOutputs, std::function()> mkDrvGoal) { - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { + std::weak_ptr & goal_weak = derivationGoals[drvPath]; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { goal = mkDrvGoal(); - abstract_goal_weak = goal; + goal_weak = goal; wakeUp(goal); } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); goal->addWantedOutputs(wantedOutputs); } return goal; @@ -73,10 +70,10 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME + std::weak_ptr & goal_weak = substitutionGoals[path]; + auto goal = goal_weak.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; @@ -85,14 +82,14 @@ GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, return goal; } - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); + for (typename std::map>::iterator i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; + typename std::map>::iterator j = i; ++j; goalMap.erase(i); i = j; } @@ -102,8 +99,12 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) void Worker::removeGoal(GoalPtr goal) { - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, derivationGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, substitutionGoals); + else + assert(false); if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals From eee18f88dd83be2810cd7090738e5949745dbefa Mon Sep 17 00:00:00 2001 From: volth Date: Mon, 12 Oct 2020 16:06:38 +0000 Subject: [PATCH 042/227] Handle amount of disk space saved by hard linking being negative Fixes bogus messages like "currently hard linking saves 17592186044416.00 MiB". --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7a04b2f89..bc692ca42 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -683,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - auto overhead = st.st_blocks * 512ULL; + int64_t overhead = st.st_blocks * 512ULL; printInfo("note: currently hard linking saves %.2f MiB", ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); From 5a97621d6deebaaef24c1523c42647bfa43b01f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:07:51 +0000 Subject: [PATCH 043/227] Prepare for build/*.hh headers --- src/libstore/local.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 212a725af..dfe1e2cc4 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -32,7 +32,7 @@ ifeq ($(HAVE_SECCOMP), 1) endif libstore_CXXFLAGS += \ - -I src/libutil -I src/libstore \ + -I src/libutil -I src/libstore -I src/libstore/build \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ @@ -64,3 +64,6 @@ $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) + +$(foreach i, $(wildcard src/libstore/build/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/build, 0644))) From f7099965bfa1ec00cce28f6ed0d5daf3bc093759 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:08:52 +0000 Subject: [PATCH 044/227] Change .cc files to use split build headers --- src/libstore/build/derivation-goal.cc | 17 ++++++++++++++++- src/libstore/build/goal.cc | 3 ++- src/libstore/build/hook-instance.cc | 3 ++- src/libstore/build/local-store-build.cc | 5 ++++- src/libstore/build/substitution-goal.cc | 3 ++- src/libstore/build/worker.cc | 6 +++++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bb8921759..241e52116 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,4 +1,6 @@ -#include "build.hh" +#include "derivation-goal.hh" +#include "hook-instance.hh" +#include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" #include "references.hh" @@ -12,6 +14,9 @@ #include "topo-sort.hh" #include "callback.hh" +#include +#include + #include #include #include @@ -140,6 +145,16 @@ DerivationGoal::~DerivationGoal() } +string DerivationGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); +} + + inline bool DerivationGoal::needsHashRewrite() { #if __linux__ diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 404a430f7..2dd7a4d37 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,4 +1,5 @@ -#include "build.hh" +#include "goal.hh" +#include "worker.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index a33d5e22b..0f6f580be 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,4 +1,5 @@ -#include "build.hh" +#include "globals.hh" +#include "hook-instance.hh" namespace nix { diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc index 407979b1e..a05fb5805 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/local-store-build.cc @@ -1,4 +1,7 @@ -#include "build.hh" +#include "machines.hh" +#include "worker.hh" +#include "substitution-goal.hh" +#include "derivation-goal.hh" namespace nix { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index dd2ce94a6..d16584f65 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4 +1,5 @@ -#include "build.hh" +#include "worker.hh" +#include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2fc9f6982..5c3fe2f57 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,4 +1,8 @@ -#include "build.hh" +#include "machines.hh" +#include "worker.hh" +#include "substitution-goal.hh" +#include "derivation-goal.hh" +#include "hook-instance.hh" #include From 4eb8c698538e0c212214d22a3c516777d52f921b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:15:32 +0000 Subject: [PATCH 045/227] Rename to hand-hold git (derivation-goal.hh) --- src/libstore/{build.hh => build/derivation-goal.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.hh => build/derivation-goal.hh} (100%) diff --git a/src/libstore/build.hh b/src/libstore/build/derivation-goal.hh similarity index 100% rename from src/libstore/build.hh rename to src/libstore/build/derivation-goal.hh From 2ce726947adf76e558bd0d05f8e309c827a46b2c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:15:32 +0000 Subject: [PATCH 046/227] Trim derivation-goal.hh --- src/libstore/build/derivation-goal.hh | 409 +------------------------- 1 file changed, 2 insertions(+), 407 deletions(-) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 8027c61b1..4976207e0 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,312 +1,18 @@ #pragma once -#include "machines.hh" #include "parsed-derivations.hh" #include "lock.hh" #include "local-store.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "goal.hh" namespace nix { using std::map; - -/* Forward definition. */ -class Worker; struct HookInstance; - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -class SubstitutionGoal; - /* Unless we are repairing, we don't both to test validity and just assume it, so the choices are `Absent` or `Valid`. */ enum struct PathStatus { @@ -554,14 +260,7 @@ public: void timedOut(Error && ex) override; - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } + string key() override; void work() override; @@ -677,108 +376,4 @@ private: StorePathSet exportReferences(const StorePathSet & storePaths); }; -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p); - } From 0d0e345cdc1a12cdd8b14a240d0ecd94269b26a5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:00 +0000 Subject: [PATCH 047/227] Rename to hand-hold git (goal.hh) --- src/libstore/{build.hh => build/goal.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.hh => build/goal.hh} (100%) diff --git a/src/libstore/build.hh b/src/libstore/build/goal.hh similarity index 100% rename from src/libstore/build.hh rename to src/libstore/build/goal.hh From 8067d32f2a69e8822e80fcf5344d8bc981952b04 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:00 +0000 Subject: [PATCH 048/227] Trim goal.hh --- src/libstore/build/goal.hh | 685 +------------------------------------ 1 file changed, 4 insertions(+), 681 deletions(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 8027c61b1..360c160ce 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,34 +1,15 @@ #pragma once -#include "machines.hh" -#include "parsed-derivations.hh" -#include "lock.hh" -#include "local-store.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "types.hh" +#include "store-api.hh" namespace nix { -using std::map; - - /* Forward definition. */ -class Worker; -struct HookInstance; - +struct Goal; +struct Worker; /* A pointer to a goal. */ -struct Goal; -class DerivationGoal; typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -43,8 +24,6 @@ typedef list WeakGoals; /* A map of paths to goals (and the other way around). */ typedef std::map WeakGoalMap; - - struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; @@ -123,662 +102,6 @@ struct Goal : public std::enable_shared_from_this void amDone(ExitCode result, std::optional ex = {}); }; -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - void addToWeakGoals(WeakGoals & goals, GoalPtr p); } From 10b749a156d501658f1138c002eeabafeb62dc53 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:13 +0000 Subject: [PATCH 049/227] Rename to hand-hold git (hook-instance.hh) --- src/libstore/{build.hh => build/hook-instance.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.hh => build/hook-instance.hh} (100%) diff --git a/src/libstore/build.hh b/src/libstore/build/hook-instance.hh similarity index 100% rename from src/libstore/build.hh rename to src/libstore/build/hook-instance.hh From d585b4c54f5a1b4d24168c5c3e7e0f3c4e903b85 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:13 +0000 Subject: [PATCH 050/227] Trim hook-instance.hh --- src/libstore/build/hook-instance.hh | 757 +--------------------------- 1 file changed, 2 insertions(+), 755 deletions(-) diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 8027c61b1..9e8cff128 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -1,760 +1,10 @@ #pragma once -#include "machines.hh" -#include "parsed-derivations.hh" -#include "lock.hh" -#include "local-store.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "logging.hh" +#include "serialise.hh" namespace nix { -using std::map; - - -/* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - struct HookInstance { /* Pipes for talking to the build hook. */ @@ -778,7 +28,4 @@ struct HookInstance ~HookInstance(); }; - -void addToWeakGoals(WeakGoals & goals, GoalPtr p); - } From e77a2344d58ef724289c07bd64c1c2e9c69187c2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:25 +0000 Subject: [PATCH 051/227] Rename to hand-hold git (substitution-goal.hh) --- src/libstore/{build.hh => build/substitution-goal.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.hh => build/substitution-goal.hh} (100%) diff --git a/src/libstore/build.hh b/src/libstore/build/substitution-goal.hh similarity index 100% rename from src/libstore/build.hh rename to src/libstore/build/substitution-goal.hh From 3ffa3546bd41d73cd69424f9c9f862988e2e6afa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:25 +0000 Subject: [PATCH 052/227] Trim substitution-goal.hh --- src/libstore/build/substitution-goal.hh | 699 +----------------------- 1 file changed, 2 insertions(+), 697 deletions(-) diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 8027c61b1..3ae9a9e6b 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,681 +1,12 @@ #pragma once -#include "machines.hh" -#include "parsed-derivations.hh" #include "lock.hh" -#include "local-store.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "store-api.hh" +#include "goal.hh" namespace nix { -using std::map; - - -/* Forward definition. */ class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; -class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; - -typedef std::chrono::time_point steady_time_point; - - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - set fds; - bool respectTimeouts; - bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ - steady_time_point timeStarted; -}; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - std::list children; - - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ - unsigned int nrLocalBuilds; - - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - - /* Goals waiting for busy paths to be unlocked. */ - WeakGoals waitingForAnyGoal; - - /* Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAWhile; - - /* Last time the goals in `waitingForAWhile' where woken up. */ - steady_time_point lastWokenUp; - - /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ - bool permanentFailure; - - /* Set if at least one derivation had a timeout. */ - bool timedOut; - - /* Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; - - /* Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; - - LocalStore & store; - - std::unique_ptr hook; - - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; - - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; - - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ - bool tryBuildHook = true; - - Worker(LocalStore & store); - ~Worker(); - - /* Make a goal (with caching). */ - - /* derivation goal */ -private: - std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, - std::function()> mkDrvGoal); -public: - std::shared_ptr makeDerivationGoal( - const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal( - const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - - /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ - unsigned int getNrLocalBuilds(); - - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set & fds, - bool inBuildSlot, bool respectTimeouts); - - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ - void childTerminated(Goal * goal, bool wakeSleepers = true); - - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ - void waitForBuildSlot(GoalPtr goal); - - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ - void waitForAnyGoal(GoalPtr goal); - - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ - void waitForAWhile(GoalPtr goal); - - /* Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); - - unsigned int exitStatus(); - - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const StorePath & path); - - void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } -}; - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; class SubstitutionGoal : public Goal { @@ -755,30 +86,4 @@ public: StorePath getStorePath() { return storePath; } }; -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p); - } From 0e2306204aa2ced6583385016dcceda112becb74 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:48 +0000 Subject: [PATCH 053/227] Rename to hand-hold git (worker.hh) --- src/libstore/{build.hh => build/worker.hh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/libstore/{build.hh => build/worker.hh} (100%) diff --git a/src/libstore/build.hh b/src/libstore/build/worker.hh similarity index 100% rename from src/libstore/build.hh rename to src/libstore/build/worker.hh From 542972f02960b3ea7f4981e9ec8d89d3c0de957d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 17:16:48 +0000 Subject: [PATCH 054/227] Trim worker.hh --- src/libstore/build/worker.hh | 597 +---------------------------------- 1 file changed, 4 insertions(+), 593 deletions(-) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 8027c61b1..a54316343 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,127 +1,14 @@ #pragma once -#include "machines.hh" -#include "parsed-derivations.hh" +#include "types.hh" #include "lock.hh" #include "local-store.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "goal.hh" namespace nix { -using std::map; - - /* Forward definition. */ -class Worker; -struct HookInstance; - - -/* A pointer to a goal. */ -struct Goal; class DerivationGoal; -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/* Set of goals. */ -typedef set Goals; -typedef list WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef std::map WeakGoalMap; - - - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ - unsigned int nrNoSubstituters; - - /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ - unsigned int nrIncompleteClosure; - - /* Name of this goal for debugging purposes. */ - string name; - - /* Whether the goal is finished. */ - ExitCode exitCode; - - /* Exception containing an error message, if any. */ - std::optional ex; - - Goal(Worker & worker) : worker(worker) - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, const string & data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(const FormatOrString & fs); - - string getName() - { - return name; - } - - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ - virtual void timedOut(Error && ex) = 0; - - virtual string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); -}; typedef std::chrono::time_point steady_time_point; @@ -140,6 +27,8 @@ struct Child steady_time_point timeStarted; }; +/* Forward definition. */ +struct HookInstance; /* The worker class. */ class Worker @@ -303,482 +192,4 @@ public: } }; -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -class SubstitutionGoal; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - std::optional known; -}; - -class DerivationGoal : public Goal -{ -private: - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* The build hook. */ - std::unique_ptr hook; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* The sort of derivation we are building. */ - DerivationType derivationType; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - -public: - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - void timedOut(Error && ex) override; - - string key() override - { - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); - } - - void work() override; - - StorePath getDrvPath() - { - return drvPath; - } - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - -private: - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - void tryLocalBuild(); - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); - - friend int childEntry(void *); - - /* Check that the derivation outputs all exist and register them - as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); - - /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); - - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -}; - -class SubstitutionGoal : public Goal -{ - friend class Worker; - -private: - /* The store path that should be realised through a substitute. */ - StorePath storePath; - - /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ - RepairFlag repair; - - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; - - std::unique_ptr> maintainExpectedSubstitutions, - maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ - std::optional ca; - -public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); - - void timedOut(Error && ex) override { abort(); }; - - string key() override - { - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - void work() override; - - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } -}; - -struct HookInstance -{ - /* Pipes for talking to the build hook. */ - Pipe toHook; - - /* Pipe for the hook's standard output/error. */ - Pipe fromHook; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* The process ID of the hook. */ - Pid pid; - - FdSink sink; - - std::map activities; - - HookInstance(); - - ~HookInstance(); -}; - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p); - } From 55592b253f3dddb121c1072ca584e95c37729b6d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 18:04:24 +0000 Subject: [PATCH 055/227] Add some more docs --- src/libstore/build/worker.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 07c0c0f16..f8bacc514 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -11,6 +11,11 @@ namespace nix { class DerivationGoal; class SubstitutionGoal; +/* Workaround for not being able to declare a something like + + class SubstitutionGoal : public Goal; + + even when Goal is a complete type; */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; From 11882d7c7ce3b6dc51dd7c0536f9662dc254ac0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Oct 2020 12:20:58 +0200 Subject: [PATCH 056/227] Create /etc/passwd *after* figuring out the sandbox uid/gid Fixes build failures like # nix log /nix/store/gjaa0psfcmqvw7ivggsncx9w364p3s8s-sshd.conf-validated.drv No user exists for uid 30012 --- src/libstore/build/derivation-goal.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 241e52116..fda05f0e9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1420,12 +1420,6 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", @@ -1730,6 +1724,14 @@ void DerivationGoal::startBuilder() throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); } + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); From f6ed1a96b397f0345af029127cfde86bcd0247d2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 18:54:36 +0000 Subject: [PATCH 057/227] `build-static` -> `buildStatic` in Nix's flake --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index a5710f608..0602861fa 100644 --- a/flake.nix +++ b/flake.nix @@ -228,7 +228,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); + buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -441,7 +441,7 @@ binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { - build-static = self.hydraJobs.build-static.${system}; + buildStatic = self.hydraJobs.buildStatic.${system}; }); packages = forAllSystems (system: { From 64be1c15c229facfa849f5667f603cce951a8488 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 19:05:06 +0000 Subject: [PATCH 058/227] Add missing include for MAX_PATH And remove one that we didn't actually need to add --- src/libstore/globals.cc | 1 - src/libutil/tests/tests.cc | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f4a4f348f..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 8e77ccbe1..ffba832d8 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "types.hh" +#include #include namespace nix { From 48ce62737750215208804947a1509ad7d26f6214 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 20:13:01 +0000 Subject: [PATCH 059/227] Make a better -lz hack Per the comments, the underlying issue is https://github.com/libarchive/libarchive/issues/1446, knowing this allows the hack to be much more targetted. --- configure.ac | 4 ++++ mk/programs.mk | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index eecb107d7..39306b953 100644 --- a/configure.ac +++ b/configure.ac @@ -179,6 +179,10 @@ AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) +# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed +if test "$shared" != yes; then + LIBARCHIVE_LIBS+=' -lz' +fi # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) diff --git a/mk/programs.mk b/mk/programs.mk index a96ff56af..3fa9685c3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) From 257090d030508160bb380554433f0c3622470c17 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 21:49:49 +0000 Subject: [PATCH 060/227] Bump Nixpkgs to hopefully fix linkrot --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 822a73332..ddff58979 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1591633336, - "narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=", + "lastModified": 1602604700, + "narHash": "sha256-TSfAZX0czPf1P8xnnGFXcoeoM9I5CaFjAdNP63W9DCY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "70717a337f7ae4e486ba71a500367cad697e5f09", + "rev": "3a10a004bb5802d5f23c58886722e4239705e733", "type": "github" }, "original": { From 5cfdf16dd62dcfa2eb32b4e9ae390cb99d683907 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 17 Oct 2020 22:08:18 +0200 Subject: [PATCH 061/227] Convert VM tests to Python Perl-based tests are deprecated since NixOS 20.03 and subsequently got removed in NixOS 20.09, which effectively means that tests are going to fail as soon as we build it with NixOS 20.09 or anything newer. I've put "# fmt: off" at the start of every testScript, because formatting with Black really messes up indentation and I don't think it really adds anything in value or readability for inlined Python scripts. Signed-off-by: aszlig --- tests/github-flakes.nix | 45 +++++------ tests/nix-copy-closure.nix | 69 +++++++++-------- tests/remote-builds.nix | 70 ++++++++--------- tests/setuid.nix | 149 ++++++++++++++++++++----------------- 4 files changed, 178 insertions(+), 155 deletions(-) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index a47610d9a..2de3e2bc0 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -1,6 +1,6 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -113,36 +113,37 @@ makeTest ( }; }; - testScript = { nodes }: - '' - use POSIX qw(strftime); + testScript = { nodes }: '' + # fmt: off + import json + import time - startAll; + start_all() - $github->waitForUnit("httpd.service"); + github.wait_for_unit("httpd.service") - $client->succeed("curl -v https://github.com/ >&2"); + client.succeed("curl -v https://github.com/ >&2") + client.succeed("nix registry list | grep nixpkgs") - $client->succeed("nix registry list | grep nixpkgs"); + rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" - $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n" - or die "revision mismatch"; + client.succeed("nix registry pin nixpkgs") - $client->succeed("nix registry pin nixpkgs"); + client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") - $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2"); + # Shut down the web server. The flake should be cached on the client. + github.succeed("systemctl stop httpd.service") - # Shut down the web server. The flake should be cached on the client. - $github->succeed("systemctl stop httpd.service"); + info = json.loads(client.succeed("nix flake info nixpkgs --json")) + date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) + assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" - my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified"); - strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModifiedDate}" or die "time mismatch"; + client.succeed("nix build nixpkgs#hello") - $client->succeed("nix build nixpkgs#hello"); - - # The build shouldn't fail even with --tarball-ttl 0 (the server - # being down should not be a fatal error). - $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0"); - ''; + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") + ''; }) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 9c9d119b7..68f9c70b3 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -25,41 +25,46 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the server. - $server->succeed("mkdir -m 700 /root/.ssh"); - $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $server->waitForUnit("sshd"); - $client->waitForUnit("network.target"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'"); + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) - # Copy the closure of package A from the client to the server. - $server->fail("nix-store --check-validity ${pkgA}"); - $client->succeed("nix-copy-closure --to server --gzip ${pkgA} >&2"); - $server->succeed("nix-store --check-validity ${pkgA}"); + client.succeed("mkdir -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Copy the closure of package B from the server to the client. - $client->fail("nix-store --check-validity ${pkgB}"); - $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2"); - $client->succeed("nix-store --check-validity ${pkgB}"); + # Install the SSH key on the server. + server.succeed("mkdir -m 700 /root/.ssh") + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") - # Copy the closure of package C via the SSH substituter. - $client->fail("nix-store -r ${pkgC}"); - # FIXME - #$client->succeed( - # "nix-store --option use-ssh-substituter true" - # . " --option ssh-substituter-hosts root\@server" - # . " -r ${pkgC} >&2"); - #$client->succeed("nix-store --check-validity ${pkgC}"); - ''; + # Copy the closure of package A from the client to the server. + server.fail("nix-store --check-validity ${pkgA}") + client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2") + server.succeed("nix-store --check-validity ${pkgA}") + # Copy the closure of package B from the server to the client. + client.fail("nix-store --check-validity ${pkgB}") + client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the closure of package C via the SSH substituter. + client.fail("nix-store -r ${pkgC}") + # FIXME + # client.succeed( + # "nix-store --option use-ssh-substituter true" + # " --option ssh-substituter-hosts root\@server" + # " -r ${pkgC} >&2" + # ) + # client.succeed("nix-store --check-validity ${pkgC}") + ''; }) diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 153956619..305c82394 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -66,44 +66,46 @@ in }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -p -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the builders. - $client->waitForUnit("network.target"); - foreach my $builder ($builder1, $builder2) { - $builder->succeed("mkdir -p -m 700 /root/.ssh"); - $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $builder->waitForUnit("sshd"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); - } + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Perform a build and check that it was performed on the builder. - my $out = $client->succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ); - $builder1->succeed("test -e $out"); + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - # And a parallel build. - my ($out1, $out2) = split /\s/, - $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); - $builder1->succeed("test -e $out1 -o -e $out2"); - $builder2->succeed("test -e $out1 -o -e $out2"); + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") - # And a failing build. - $client->fail("nix-build ${expr nodes.client.config 5}"); + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") - # Test whether the build hook automatically skips unavailable builders. - $builder1->block; - $client->succeed("nix-build ${expr nodes.client.config 4}"); - ''; + # And a failing build. + client.fail("nix-build ${expr nodes.client.config 5}") + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client.config 4}") + ''; }) diff --git a/tests/setuid.nix b/tests/setuid.nix index 6f2f7d392..8d31a18be 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -17,94 +17,109 @@ makeTest { virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + start_all() - # Copying to /tmp should succeed. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - ")\' '); + # Copying to /tmp should succeed. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setuid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 4755 /tmp/id - ")\' '); + # Creating a setuid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setgid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # Creating a setgid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The checks should also work on 32-bit binaries. - $machine->fail('nix-build --no-sandbox -E \'(with import { system = "i686-linux"; }; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # The checks should also work on 32-bit binaries. + machine.fail(r""" + nix-build --no-sandbox -E '(with import { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The tests above use fchmodat(). Test chmod() as well. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 0666, qw(/tmp/id) or die\" - ")\' '); + # The tests above use fchmodat(). Test chmod() as well. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 04755, qw(/tmp/id) or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # And test fchmod(). - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" - ")\' '); + # And test fchmod(). + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); - - $machine->succeed("rm /tmp/id"); - ''; + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + machine.succeed("rm /tmp/id") + ''; } From cfa26cf18135ec1a16ce72319b0bc8b297600ef1 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 17 Oct 2020 23:34:38 +0200 Subject: [PATCH 062/227] tests: Add names to VM tests Having vm-test-run-unnamed for all the test derivation doesn't look very nice, so in order to better distinguish them from their store path, let's actually give them proper names. Signed-off-by: aszlig --- tests/github-flakes.nix | 1 + tests/nix-copy-closure.nix | 1 + tests/remote-builds.nix | 1 + tests/setuid.nix | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index 2de3e2bc0..7ac397d81 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -64,6 +64,7 @@ in makeTest ( { + name = "github-flakes"; nodes = { # Impersonate github.com and api.github.com. diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 68f9c70b3..e5f6a0f12 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { }; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { + name = "nix-copy-closure"; nodes = { client = diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 305c82394..b9e7352c0 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -36,6 +36,7 @@ let in { + name = "remote-builds"; nodes = { builder1 = builder; diff --git a/tests/setuid.nix b/tests/setuid.nix index 8d31a18be..35eb304ed 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { }; makeTest { + name = "setuid"; machine = { config, lib, pkgs, ... }: From e6f8ae56d82813edbcf09fda58305de47369f964 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:45:31 +0000 Subject: [PATCH 063/227] tab -> space --- src/libstore/build/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 97ffb9a34..4d3df26f3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -455,7 +455,7 @@ void Worker::markContentsGood(const StorePath & path) GoalPtr upcast_goal(std::shared_ptr subGoal) { - return subGoal; + return subGoal; } } From 57d0432b395cc4d70792d2df5794ff2e0dd02d3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:47:52 +0000 Subject: [PATCH 064/227] Just use `auto` in two places. --- src/libstore/build/worker.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4d3df26f3..17c10cd71 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -90,10 +90,10 @@ template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (typename std::map>::iterator i = goalMap.begin(); + for (auto i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - typename std::map>::iterator j = i; ++j; + auto j = i; ++j; goalMap.erase(i); i = j; } From 7ed46c15744461534478e2ed0aa25a2b2e536c6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:50:12 +0000 Subject: [PATCH 065/227] Explain that `upcast_goal` is still a static cast --- src/libstore/build/worker.hh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index f8bacc514..3a53a8def 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -15,7 +15,11 @@ class SubstitutionGoal; class SubstitutionGoal : public Goal; - even when Goal is a complete type; */ + even when Goal is a complete type. + + This is still a static cast. The purpose of exporting it is to define it in + a place where `SubstitutionGoal` is concrete, and use it in a place where it + is opaque. */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; From a53438d18f1faddb9f15c6980c828eb11820284c Mon Sep 17 00:00:00 2001 From: Stefan Frijters Date: Sun, 18 Oct 2020 14:14:37 +0200 Subject: [PATCH 066/227] doc: nix-shell in pure mode does *not* source user bashrc --- doc/manual/src/command-ref/nix-shell.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 45a5ff08c..d1266930e 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -76,8 +76,8 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. Note that `~/.bashrc` and (depending on your Bash - installation) `/etc/bashrc` are still sourced, so any variables set + retained. Note that (depending on your Bash + installation) `/etc/bashrc` is still sourced, so any variables set there will affect the interactive shell. - `--packages` / `-p` *packages*… From 94f1e4a441ee64bcc7a961d941ec901de750d880 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:26:23 +0200 Subject: [PATCH 067/227] Typo --- src/libstore/build/goal.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 360c160ce..0781a9d38 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this unsigned int nrNoSubstituters; /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ + failed because they had unsubstitutable references. */ unsigned int nrIncompleteClosure; /* Name of this goal for debugging purposes. */ From ea8d32020e1e24576cf5e8ff1a835f149dbd81c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 00:43:52 +0200 Subject: [PATCH 068/227] Tests for #3964 --- tests/binary-cache.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index fe4ddec8d..e14cf882e 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -239,3 +239,37 @@ nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath diff -u \ <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) + +# Test against issue https://github.com/NixOS/nix/issues/3964 +# +expr=' + with import ./config.nix; + mkDerivation { + name = "multi-output"; + buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref"; + outputs = ["out" "doc"]; + } +' +outPath=$(nix-build --no-out-link -E "$expr") +docPath=$(nix-store -q --references $outPath) + +# $ nix-store -q --tree $outPath +# ...-multi-output +# +---...-multi-output-doc + +nix copy --to "file://$cacheDir" $outPath +( echo $outPath $docPath + find $cacheDir +) >/tmp/blurb + +hashpart() { + basename "$1" | cut -c1-32 +} + +# break the closure of out by removing doc +rm $cacheDir/$(hashpart $docPath).narinfo + +nix-store --delete $outPath $docPath +# -vvv is the level that logs during the loop +timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ + --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From bd9eb5c743faf1b3c33f4e1c2ccf317977d4be9d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:21:53 +0200 Subject: [PATCH 069/227] DerivationGoal: only retry if output closure incomplete is only problem --- src/libstore/build/derivation-goal.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fda05f0e9..1c9217537 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -330,8 +330,13 @@ void DerivationGoal::outputsSubstitutionTried() /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; From 93bd014c8c416580d8eeaa487e180d6c2549028c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 20:32:46 +0200 Subject: [PATCH 070/227] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index b087cd8d5..c51582cf0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-copy-closure /scripts/nix-reduce-build /scripts/nix-http-export.cgi /scripts/nix-profile-daemon.sh From 20a7d8d23a19530207378fc74ddd558232240f43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 20:32:59 +0200 Subject: [PATCH 071/227] Add some missing clean-files --- misc/systemd/local.mk | 2 ++ misc/upstart/local.mk | 2 ++ 2 files changed, 4 insertions(+) diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 004549fd2..785db52a4 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -2,4 +2,6 @@ ifeq ($(OS), Linux) $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) + clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service + endif diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index a73dc061e..5071676dc 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -2,4 +2,6 @@ ifeq ($(OS), Linux) $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) + clean-files += $(d)/nix-daemon.conf + endif From 532d2bc1890d59a31e212eaea8d351a8cca66e19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:20:35 +0200 Subject: [PATCH 072/227] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3a10a004bb5802d5f23c58886722e4239705e733' -> 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31' --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index ddff58979..9f8c788ac 100644 --- a/flake.lock +++ b/flake.lock @@ -18,16 +18,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1602604700, - "narHash": "sha256-TSfAZX0czPf1P8xnnGFXcoeoM9I5CaFjAdNP63W9DCY=", + "lastModified": 1602702596, + "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a10a004bb5802d5f23c58886722e4239705e733", + "rev": "ad0d20345219790533ebe06571f82ed6b034db31", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-20.03-small", + "ref": "nixos-20.09-small", "type": "indirect" } }, From 62cf1d815a6c55853e9a4041e85b7e6e0b04d7e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:31:27 +0200 Subject: [PATCH 073/227] Switch to Nixpkgs 20.09 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 0602861fa..22ea2911e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small"; + inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: From c27fcd94ce3de918d506afe90759d97d1946eca1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:44:07 +0200 Subject: [PATCH 074/227] Remove buildStatic from checks checks should be relatively fast, but buildStatic depends on a lot of stuff that isn't in the binary cache (e.g. musl builds of Git and Mercurial that we probably don't need since we don't link against them...). --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 22ea2911e..abc614c8d 100644 --- a/flake.nix +++ b/flake.nix @@ -440,8 +440,6 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; - } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { - buildStatic = self.hydraJobs.buildStatic.${system}; }); packages = forAllSystems (system: { From 9c3dc9d7ca11555fbafe77e9ca8ed9fc214ab2f8 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 7 Sep 2020 23:53:31 -0500 Subject: [PATCH 075/227] update macOS version handling for Big Sur Keeping this commit narrow for reviewability, but some of these conditionals will change in subsequent commits in this PR. Fixes #3852. --- scripts/install-nix-from-closure.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 6efd8af18..e7e063007 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -24,9 +24,11 @@ fi # macOS support for 10.12.6 or higher if [ "$(uname -s)" = "Darwin" ]; then - macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) - macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) - if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + IFS='.' read macos_major macos_minor macos_patch << EOF +$(sw_vers -productVersion) +EOF + if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + # patch may not be present; command substitution for simplicity echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi @@ -88,7 +90,7 @@ while [ $# -gt 0 ]; do ) >&2 # darwin and Catalina+ - if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then + if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then ( echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" echo " store and mount it at /nix. This is the recommended way to create" @@ -109,7 +111,7 @@ if [ "$(uname -s)" = "Darwin" ]; then fi info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) - if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then + if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." From 1f02b65c590b5a33c1b70dba186eacfe3f67f149 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 00:07:53 -0500 Subject: [PATCH 076/227] fix xpath and conditional bugs; xpath -> xmllint - xpath -> xmllint: xpath's cli interface changed in Big Sur rather than add conditional logic for picking the correct syntax for xpath, I'm changing to xmllint --xpath, which appears to be consistent across versions I've tested... - /plist/dict/key[text()='Writable']/following-sibling::true[1] doesn't do quite what's expected. It was written to try to select a node paired with the Writable key, but it will also select the *next* node that appears even if it was paired with another key. - I think there's also a logic bug in the conditionals here. I'm not sure anyone ever actuall saw it, thanks to the xpath bug, though. With the xpath fix, this conditional passes if /nix does not exist, / IS writable, and the version is Catalina+. I think it meant to test for /nix does not exist, / is NOT writable, and the version is Catalina+. I reworked this lightly to make it a little clearer at the code level. --- scripts/install-nix-from-closure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index e7e063007..4e64f3d43 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -110,8 +110,8 @@ if [ "$(uname -s)" = "Darwin" ]; then "$self/create-darwin-volume.sh" fi - info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) - if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then + writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)" + if ! [ -e $dest ] && [ "$writable" = "false" ]; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." From e736f8f6e44180d7ed7cc1975b48c603c6c4f611 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 00:45:27 -0500 Subject: [PATCH 077/227] replace xpath with xmllint --xpath; simplify As mentioned in previous commit, Big Sur changes the syntax for the xpath command slightly. In the process of testing out replacements for these, I noticed a few small simplification wins. --- scripts/create-darwin-volume.sh | 46 ++++++--------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index dac30d72d..8c4558c7f 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -5,42 +5,13 @@ root_disk() { diskutil info -plist / } -apfs_volumes_for() { - disk=$1 - diskutil apfs list -plist "$disk" -} - -disk_identifier() { - xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null -} - -volume_list_true() { - key=$1 - xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null -} - -volume_get_string() { - key=$1 i=$2 - xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null +# i.e., "disk1" +root_disk_identifier() { + diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - } find_nix_volume() { - disk=$1 - i=1 - volumes=$(apfs_volumes_for "$disk") - while true; do - name=$(echo "$volumes" | volume_get_string "Name" "$i") - if [ -z "$name" ]; then - break - fi - case "$name" in - [Nn]ix*) - echo "$name" - break - ;; - esac - i=$((i+1)) - done + diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true } test_fstab() { @@ -89,9 +60,7 @@ test_t2_chip_present(){ } test_filevault_in_use() { - disk=$1 - # list vols on disk | get value of Filevault key | value is true - apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true + fdesetup isactive >/dev/null } # use after error msg for conditions we don't understand @@ -143,12 +112,12 @@ main() { fi fi - disk=$(root_disk | disk_identifier) + disk="$(root_disk_identifier)" volume=$(find_nix_volume "$disk") if [ -z "$volume" ]; then echo "Creating a Nix Store volume..." >&2 - if test_filevault_in_use "$disk"; then + if test_filevault_in_use; then # TODO: Not sure if it's in-scope now, but `diskutil apfs list` # shows both filevault and encrypted at rest status, and it # may be the more semantic way to test for this? It'll show @@ -178,6 +147,7 @@ main() { if ! test_fstab; then echo "Configuring /etc/fstab..." >&2 label=$(echo "$volume" | sed 's/ /\\040/g') + # shellcheck disable=SC2209 printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs fi } From fe807904e5e6e56b551f34f3586e69ea6498287c Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 01:01:11 -0500 Subject: [PATCH 078/227] adapt to apfs.util flag diff in catalina/big sur Fixes #3957. Just runs both forms to minimize moving parts. --- scripts/create-darwin-volume.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 8c4558c7f..32fa577a8 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -26,6 +26,20 @@ test_synthetic_conf() { grep -q "^nix$" /etc/synthetic.conf 2>/dev/null } +# Create the paths defined in synthetic.conf, saving us a reboot. +create_synthetic_objects(){ + # Big Sur takes away the -B flag we were using and replaces it + # with a -t flag that appears to do the same thing (but they + # don't behave exactly the same way in terms of return values). + # This feels a little dirty, but as far as I can tell the + # simplest way to get the right one is to just throw away stderr + # and call both... :] + { + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true # Big Sur + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true # Catalina + } >/dev/null 2>&1 +} + test_nix() { test -d "/nix" } @@ -101,7 +115,7 @@ main() { if ! test_nix; then echo "Creating mountpoint for /nix..." >&2 - /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true + create_synthetic_objects # the ones we defined in synthetic.conf if ! test_nix; then sudo mkdir -p /nix 2>/dev/null || true fi From 3a8699ac4ffc0d9b611c471e2668e6b22cc65767 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 10 Sep 2020 18:21:04 -0500 Subject: [PATCH 079/227] restore create-darwin-volume to release tarball The move from release.nix to flake.nix appears to have lost some changes from #3628 / 1c56f18a8122b605c28000e295d5e223f272cccd, leaving create-darwin-volume.sh out of the release tarball. Under the assumption that this was just an accident/byproduct of when flake.nix split off and not intentional, I am restoring those edits. --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index a50533a29..d0c3bfcd2 100644 --- a/flake.nix +++ b/flake.nix @@ -235,6 +235,7 @@ } '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} @@ -253,6 +254,7 @@ # SC1090: Don't worry about not being able to find # $nix/etc/profile.d/nix.sh shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh shellcheck $TMPDIR/install-darwin-multi-user.sh shellcheck $TMPDIR/install-systemd-multi-user.sh @@ -268,6 +270,7 @@ fi chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh chmod +x $TMPDIR/install-darwin-multi-user.sh chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user @@ -280,11 +283,15 @@ --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ $(cat ${installerClosureInfo}/store-paths) ''); From b719f686a8c8936fe831ce730f28638d8b1e2982 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 11 Sep 2020 12:06:01 -0500 Subject: [PATCH 080/227] fix skipped multi-user install steps on macOS Some of the changes in #3788 to support non-systemd Nix installs don't appear to be aware that the darwin installer exists, which resulted in some skipped steps and inappropriate instructions. --- scripts/install-darwin-multi-user.sh | 7 +++++ scripts/install-multi-user.sh | 34 +++----------------- scripts/install-systemd-multi-user.sh | 45 ++++++++++++++++++++------- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 49076bd5c..a27be2a43 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -37,6 +37,13 @@ poly_service_setup_note() { EOF } +poly_extra_try_me_commands(){ + : +} +poly_extra_setup_instructions(){ + : +} + poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon as a LaunchDaemon" \ cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index e5cc4d7ed..54edfe40d 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -71,11 +71,9 @@ uninstall_directions() { subheader "Uninstalling nix:" local step=0 - if [ -e /run/systemd/system ] && poly_service_installed_check; then + if poly_service_installed_check; then step=$((step + 1)) poly_service_uninstall_directions "$step" - else - step=$((step + 1)) fi for profile_target in "${PROFILE_TARGETS[@]}"; do @@ -255,40 +253,20 @@ function finish_success { echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." fi - if [ -e /run/systemd/system ]; then - cat < Date: Fri, 11 Sep 2020 16:45:58 -0500 Subject: [PATCH 081/227] create missing profile files to fix zsh envvars Env vars for ZSH were moved from /etc/zshrc to /etc/zshenv in #3608 to address an issue with zshrc getting clobbered by OS updates, but /etc/zshenv doesn't exist by default--so *nothing* would get set up for zsh users unless they already happened to have /etc/zshenv. Creating these files if they don't exist. Also cut separate creation of profile.d/nix.sh, which isn't needed now. --- scripts/install-multi-user.sh | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 54edfe40d..5e8b4ac18 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -608,24 +608,20 @@ EOF } configure_shell_profile() { - # If there is an /etc/profile.d directory, we want to ensure there - # is a nix.sh within it, so we can use the following loop to add - # the source lines to it. Note that I'm _not_ adding the source - # lines here, because we want to be using the regular machinery. - # - # If we go around that machinery, it becomes more complicated and - # adds complications to the uninstall instruction generator and - # old instruction sniffer as well. - if [ -d /etc/profile.d ]; then - _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \ - touch /etc/profile.d/nix.sh - fi - for profile_target in "${PROFILE_TARGETS[@]}"; do if [ -e "$profile_target" ]; then _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \ cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX" + else + # try to create the file if its directory exists + target_dir="$(dirname "$profile_target")" + if [ -d "$target_dir" ]; then + _sudo "to create a stub $profile_target which will be updated" \ + touch "$profile_target" + fi + fi + if [ -e "$profile_target" ]; then shell_source_lines \ | _sudo "extend your $profile_target with nix-daemon settings" \ tee -a "$profile_target" From f289bdb9d4e8432160c5dcdc037a930b2025d11b Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 19 Oct 2020 11:54:21 -0500 Subject: [PATCH 082/227] discourage casual Big Sur installs --- scripts/install-nix-from-closure.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 4e64f3d43..ea2e47b7f 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -27,6 +27,15 @@ if [ "$(uname -s)" = "Darwin" ]; then IFS='.' read macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF + # TODO: this is a temporary speed-bump to keep people from naively installing Nix + # on macOS Big Sur (11.0+, 10.16+) until nixpkgs updates are ready for them. + # *Ideally* this is gone before next Nix release. If you're intentionally working on + # Nix + Big Sur, just comment out this block and be on your way :) + if [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 15 ]; }; then + echo "$0: nixpkgs isn't quite ready to support macOS $(sw_vers -productVersion) yet" + exit 1 + fi + if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then # patch may not be present; command substitution for simplicity echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" From f6aaac2b5983bd87cd6e2f648c9dd835c6d9373c Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Tue, 20 Oct 2020 11:48:07 -0500 Subject: [PATCH 083/227] Make bash non-interactive for nix develop --phase Fix #3975: Currently if Ctrl-C is pressed during a phase, the interactive subshell is not exited. Removing --rcfile when --phase is present makes bash non-interactive --- src/nix/develop.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9372f43de..380417c82 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -368,7 +368,6 @@ struct CmdDevelop : Common, MixEnvironment // rid of that. script += fmt("foundMakefile=1\n"); script += fmt("runHook %1%Phase\n", *phase); - script += fmt("exit 0\n", *phase); } else if (!command.empty()) { @@ -408,7 +407,10 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } - auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + // If running a phase, don't want an interactive shell running after + // Ctrl-C, so don't pass --rcfile + auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath} + : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); restoreSignals(); From 39fbd3d82823ae6eadfbf91bb2b356caef90841c Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 21 Oct 2020 00:03:38 +0400 Subject: [PATCH 084/227] Fix iterating over $NIX_PROFILES in Zsh NIX_PROFILES is space separated list of directories, and passing it into for as is is considered to be 1-element list with the whole string. With shwordsplit option Zsh emulates other shells in this regard ans implicitely splits unquoted strings into words. Fixes #4167. --- scripts/nix-profile-daemon.sh.in | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 4bc7c6fc5..500a98992 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -17,11 +17,21 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES; do - if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then - export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + check_nix_profiles() { + if [ "$ZSH_VERSION" ]; then + # Zsh by default doesn't split words in unquoted parameter expansion. + # Set local_options for these options to be reverted at the end of the function + # and shwordsplit to force splitting words in $NIX_PROFILES below. + setopt local_options shwordsplit fi - done + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done + } + check_nix_profiles + unset -f check_nix_profiles fi export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" From 461cf2b85601e4510bff303e454059d80b9df8c0 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Mon, 19 Oct 2020 23:08:50 +0200 Subject: [PATCH 085/227] Add NIX_CONFIG env var for applying nix.conf overrides --- doc/manual/src/command-ref/conf-file-prefix.md | 3 +++ doc/manual/src/command-ref/env-common.md | 5 +++++ src/libstore/globals.cc | 6 ++++++ tests/config.sh | 9 +++++++++ 4 files changed, 23 insertions(+) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 9987393d2..d38456788 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -19,6 +19,9 @@ By default Nix reads settings from the following places: and `XDG_CONFIG_HOME`. If these are unset, it will look in `$HOME/.config/nix.conf`. + - If `NIX_OPTIONS` is set, its contents is treated as the contents of + a configuration file. + The configuration files consist of `name = value` pairs, one per line. Other files can be included with a line like `include diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 03016dba7..27e730fc8 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -81,6 +81,11 @@ Most Nix commands interpret the following environment variables: Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). + - `NIX_OPTIONS` + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + - `NIX_USER_CONF_FILES` Overrides the location of the user Nix configuration files to load from (defaults to the XDG spec locations). The variable is treated diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1238dc530..4df68d0c9 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -86,6 +86,12 @@ void loadConfFile() for (auto file = files.rbegin(); file != files.rend(); file++) { globalConfig.applyConfigFile(*file); } + + auto nixConfEnv = getEnv("NIX_CONFIG"); + if (nixConfEnv.has_value()) { + globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + } + } std::vector getUserConfigFiles() diff --git a/tests/config.sh b/tests/config.sh index 8fa349f11..eaa46c395 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -16,3 +16,12 @@ here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] + +# Test that it's possible to load config from the environment +prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" +exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +[[ $prev != $exp_cores ]] +[[ $exp_cores == "4242" ]] +[[ $exp_features == "nix-command flakes" ]] \ No newline at end of file From bdf2bcc989348fdcf223c9e2a383618454453eb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 20 Oct 2020 19:18:02 +0200 Subject: [PATCH 086/227] Remove conf-file.xml This was probably revived in a bad merge. --- doc/manual/command-ref/conf-file.xml | 1236 -------------------------- 1 file changed, 1236 deletions(-) delete mode 100644 doc/manual/command-ref/conf-file.xml diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml deleted file mode 100644 index d0f1b09ca..000000000 --- a/doc/manual/command-ref/conf-file.xml +++ /dev/null @@ -1,1236 +0,0 @@ - - - - - nix.conf - 5 - Nix - - - - - nix.conf - Nix configuration file - - -Description - -By default Nix reads settings from the following places: - -The system-wide configuration file -sysconfdir/nix/nix.conf -(i.e. /etc/nix/nix.conf on most systems), or -$NIX_CONF_DIR/nix.conf if -NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The -client assumes that the daemon has already loaded them. - - -User-specific configuration files: - - - If NIX_USER_CONF_FILES is set, then each path separated by - : will be loaded in reverse order. - - - - Otherwise it will look for nix/nix.conf files in - XDG_CONFIG_DIRS and XDG_CONFIG_HOME. - - The default location is $HOME/.config/nix.conf if - those environment variables are unset. - - -The configuration files consist of -name = -value pairs, one per line. Other -files can be included with a line like include -path, where -path is interpreted relative to the current -conf file and a missing file is an error unless -!include is used instead. -Comments start with a # character. Here is an -example configuration file: - - -keep-outputs = true # Nice for developers -keep-derivations = true # Idem - - -You can override settings on the command line using the - flag, e.g. --option keep-outputs -false. - -The following settings are currently available: - - - - - allowed-uris - - - - A list of URI prefixes to which access is allowed in - restricted evaluation mode. For example, when set to - https://github.com/NixOS, builtin functions - such as fetchGit are allowed to access - https://github.com/NixOS/patchelf.git. - - - - - - - allow-import-from-derivation - - By default, Nix allows you to import from a derivation, - allowing building at evaluation time. With this option set to false, Nix will throw an error - when evaluating an expression that uses this feature, allowing users to ensure their evaluation - will not require any builds to take place. - - - - - allow-new-privileges - - (Linux-specific.) By default, builders on Linux - cannot acquire new privileges by calling setuid/setgid programs or - programs that have file capabilities. For example, programs such - as sudo or ping will - fail. (Note that in sandbox builds, no such programs are available - unless you bind-mount them into the sandbox via the - option.) You can allow the - use of such programs by enabling this option. This is impure and - usually undesirable, but may be useful in certain scenarios - (e.g. to spin up containers or set up userspace network interfaces - in tests). - - - - - allowed-users - - - - A list of names of users (separated by whitespace) that - are allowed to connect to the Nix daemon. As with the - option, you can specify groups by - prefixing them with @. Also, you can allow - all users by specifying *. The default is - *. - - Note that trusted users are always allowed to connect. - - - - - - - auto-optimise-store - - If set to true, Nix - automatically detects files in the store that have identical - contents, and replaces them with hard links to a single copy. - This saves disk space. If set to false (the - default), you can still run nix-store - --optimise to get rid of duplicate - files. - - - - - builders - - A list of machines on which to perform builds. See for details. - - - - - builders-use-substitutes - - If set to true, Nix will instruct - remote build machines to use their own binary substitutes if available. In - practical terms, this means that remote hosts will fetch as many build - dependencies as possible from their own substitutes (e.g, from - cache.nixos.org), instead of waiting for this host to - upload them all. This can drastically reduce build times if the network - connection between this computer and the remote build host is slow. Defaults - to false. - - - - build-users-group - - This options specifies the Unix group containing - the Nix build user accounts. In multi-user Nix installations, - builds should not be performed by the Nix account since that would - allow users to arbitrarily modify the Nix store and database by - supplying specially crafted builders; and they cannot be performed - by the calling user since that would allow him/her to influence - the build result. - - Therefore, if this option is non-empty and specifies a valid - group, builds will be performed under the user accounts that are a - member of the group specified here (as listed in - /etc/group). Those user accounts should not - be used for any other purpose! - - Nix will never run two builds under the same user account at - the same time. This is to prevent an obvious security hole: a - malicious user writing a Nix expression that modifies the build - result of a legitimate Nix expression being built by another user. - Therefore it is good to have as many Nix build user accounts as - you can spare. (Remember: uids are cheap.) - - The build users should have permission to create files in - the Nix store, but not delete them. Therefore, - /nix/store should be owned by the Nix - account, its group should be the group specified here, and its - mode should be 1775. - - If the build users group is empty, builds will be performed - under the uid of the Nix process (that is, the uid of the caller - if NIX_REMOTE is empty, the uid under which the Nix - daemon runs if NIX_REMOTE is - daemon). Obviously, this should not be used in - multi-user settings with untrusted users. - - - - - - - compress-build-log - - If set to true (the default), - build logs written to /nix/var/log/nix/drvs - will be compressed on the fly using bzip2. Otherwise, they will - not be compressed. - - - - connect-timeout - - - - The timeout (in seconds) for establishing connections in - the binary cache substituter. It corresponds to - curl’s - option. - - - - - - - cores - - Sets the value of the - NIX_BUILD_CORES environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - enableParallelBuilding is set to - true, the builder passes the - flag to GNU Make. - It can be overridden using the command line switch and - defaults to 1. The value 0 - means that the builder should use all available CPU cores in the - system. - - See also . - - - diff-hook - - - Absolute path to an executable capable of diffing build results. - The hook executes if is - true, and the output of a build is known to not be the same. - This program is not executed to determine if two results are the - same. - - - - The diff hook is executed by the same user and group who ran the - build. However, the diff hook does not have write access to the - store path just built. - - - The diff hook program receives three parameters: - - - - - A path to the previous build's results - - - - - - A path to the current build's results - - - - - - The path to the build's derivation - - - - - - The path to the build's scratch directory. This directory - will exist only if the build was run with - . - - - - - - The stderr and stdout output from the diff hook will not be - displayed to the user. Instead, it will print to the nix-daemon's - log. - - - When using the Nix daemon, diff-hook must - be set in the nix.conf configuration file, and - cannot be passed at the command line. - - - - - - enforce-determinism - - See . - - - - extra-sandbox-paths - - A list of additional paths appended to - . Useful if you want to extend - its default value. - - - - - extra-platforms - - Platforms other than the native one which - this machine is capable of building for. This can be useful for - supporting additional architectures on compatible machines: - i686-linux can be built on x86_64-linux machines (and the default - for this setting reflects this); armv7 is backwards-compatible with - armv6 and armv5tel; some aarch64 machines can also natively run - 32-bit ARM code; and qemu-user may be used to support non-native - platforms (though this may be slow and buggy). Most values for this - are not enabled by default because build systems will often - misdetect the target platform and generate incompatible code, so you - may wish to cross-check the results of using this option against - proper natively-built versions of your - derivations. - - - - - extra-substituters - - Additional binary caches appended to those - specified in . When used by - unprivileged users, untrusted substituters (i.e. those not listed - in ) are silently - ignored. - - - - fallback - - If set to true, Nix will fall - back to building from source if a binary substitute fails. This - is equivalent to the flag. The - default is false. - - - - fsync-metadata - - If set to true, changes to the - Nix store metadata (in /nix/var/nix/db) are - synchronously flushed to disk. This improves robustness in case - of system crashes, but reduces performance. The default is - true. - - - - hashed-mirrors - - A list of web servers used by - builtins.fetchurl to obtain files by hash. - Given a hash type ht and a base-16 hash - h, Nix will try to download the file - from - hashed-mirror/ht/h. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the hashed mirror - http://tarballs.example.com/, when building the - derivation - - -builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} - - - Nix will attempt to download this file from - http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae - first. If it is not available there, if will try the original URI. - - - - - http-connections - - The maximum number of parallel TCP connections - used to fetch files from binary caches and by other downloads. It - defaults to 25. 0 means no limit. - - - - - keep-build-log - - If set to true (the default), - Nix will write the build log of a derivation (i.e. the standard - output and error of its builder) to the directory - /nix/var/log/nix/drvs. The build log can be - retrieved using the command nix-store -l - path. - - - - - keep-derivations - - If true (default), the garbage - collector will keep the derivations from which non-garbage store - paths were built. If false, they will be - deleted unless explicitly registered as a root (or reachable from - other roots). - - Keeping derivation around is useful for querying and - traceability (e.g., it allows you to ask with what dependencies or - options a store path was built), so by default this option is on. - Turn it off to save a bit of disk space (or a lot if - keep-outputs is also turned on). - - - keep-env-derivations - - If false (default), derivations - are not stored in Nix user environments. That is, the derivations of - any build-time-only dependencies may be garbage-collected. - - If true, when you add a Nix derivation to - a user environment, the path of the derivation is stored in the - user environment. Thus, the derivation will not be - garbage-collected until the user environment generation is deleted - (nix-env --delete-generations). To prevent - build-time-only dependencies from being collected, you should also - turn on keep-outputs. - - The difference between this option and - keep-derivations is that this one is - “sticky”: it applies to any user environment created while this - option was enabled, while keep-derivations - only applies at the moment the garbage collector is - run. - - - - keep-outputs - - If true, the garbage collector - will keep the outputs of non-garbage derivations. If - false (default), outputs will be deleted unless - they are GC roots themselves (or reachable from other roots). - - In general, outputs must be registered as roots separately. - However, even if the output of a derivation is registered as a - root, the collector will still delete store paths that are used - only at build time (e.g., the C compiler, or source tarballs - downloaded from the network). To prevent it from doing so, set - this option to true. - - - max-build-log-size - - - - This option defines the maximum number of bytes that a - builder can write to its stdout/stderr. If the builder exceeds - this limit, it’s killed. A value of 0 (the - default) means that there is no limit. - - - - - - max-free - - When a garbage collection is triggered by the - min-free option, it stops as soon as - max-free bytes are available. The default is - infinity (i.e. delete all garbage). - - - - max-jobs - - This option defines the maximum number of jobs - that Nix will try to build in parallel. The default is - 1. The special value auto - causes Nix to use the number of CPUs in your system. 0 - is useful when using remote builders to prevent any local builds (except for - preferLocalBuild derivation attribute which executes locally - regardless). It can be - overridden using the () - command line switch. - - See also . - - - - max-silent-time - - - - This option defines the maximum number of seconds that a - builder can go without producing any data on standard output or - standard error. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite - loop, or to catch remote builds that are hanging due to network - problems. It can be overridden using the command - line switch. - - The value 0 means that there is no - timeout. This is also the default. - - - - - - min-free - - - When free disk space in /nix/store - drops below min-free during a build, Nix - performs a garbage-collection until max-free - bytes are available or there is no more garbage. A value of - 0 (the default) disables this feature. - - - - - narinfo-cache-negative-ttl - - - - The TTL in seconds for negative lookups. If a store path is - queried from a substituter but was not found, there will be a - negative lookup cached in the local disk cache database for the - specified duration. - - - - - - narinfo-cache-positive-ttl - - - - The TTL in seconds for positive lookups. If a store path is - queried from a substituter, the result of the query will be cached - in the local disk cache database including some of the NAR - metadata. The default TTL is a month, setting a shorter TTL for - positive lookups can be useful for binary caches that have - frequent garbage collection, in which case having a more frequent - cache invalidation would prevent trying to pull the path again and - failing with a hash mismatch if the build isn't reproducible. - - - - - - - netrc-file - - If set to an absolute path to a netrc - file, Nix will use the HTTP authentication credentials in this file when - trying to download from a remote host through HTTP or HTTPS. Defaults to - $NIX_CONF_DIR/netrc. - - The netrc file consists of a list of - accounts in the following format: - - -machine my-machine -login my-username -password my-password - - - For the exact syntax, see the - curl documentation. - - This must be an absolute path, and ~ - is not resolved. For example, ~/.netrc won't - resolve to your home directory's .netrc. - - - - - - - plugin-files - - - A list of plugin files to be loaded by Nix. Each of these - files will be dlopened by Nix, allowing them to affect - execution through static initialization. In particular, these - plugins may construct static instances of RegisterPrimOp to - add new primops or constants to the expression language, - RegisterStoreImplementation to add new store implementations, - RegisterCommand to add new subcommands to the - nix command, and RegisterSetting to add new - nix config settings. See the constructors for those types for - more details. - - - Since these files are loaded into the same address space as - Nix itself, they must be DSOs compatible with the instance of - Nix running at the time (i.e. compiled against the same - headers, not linked to any incompatible libraries). They - should not be linked to any Nix libs directly, as those will - be available already at load time. - - - If an entry in the list is a directory, all files in the - directory are loaded as plugins (non-recursively). - - - - - - pre-build-hook - - - - - If set, the path to a program that can set extra - derivation-specific settings for this system. This is used for settings - that can't be captured by the derivation model itself and are too variable - between different versions of the same system to be hard-coded into nix. - - - The hook is passed the derivation path and, if sandboxes are enabled, - the sandbox directory. It can then modify the sandbox and send a series of - commands to modify various settings to stdout. The currently recognized - commands are: - - - - extra-sandbox-paths - - - - Pass a list of files and directories to be included in the - sandbox for this build. One entry per line, terminated by an empty - line. Entries have the same format as - sandbox-paths. - - - - - - - - - - - post-build-hook - - Optional. The path to a program to execute after each build. - - This option is only settable in the global - nix.conf, or on the command line by trusted - users. - - When using the nix-daemon, the daemon executes the hook as - root. If the nix-daemon is not involved, the - hook runs as the user executing the nix-build. - - - The hook executes after an evaluation-time build. - The hook does not execute on substituted paths. - The hook's output always goes to the user's terminal. - If the hook fails, the build succeeds but no further builds execute. - The hook executes synchronously, and blocks other builds from progressing while it runs. - - - The program executes with no arguments. The program's environment - contains the following environment variables: - - - - DRV_PATH - - The derivation for the built paths. - Example: - /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv - - - - - - OUT_PATHS - - Output paths of the built derivation, separated by a space character. - Example: - /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev - /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc - /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info - /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man - /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. - - - - - - See for an example - implementation. - - - - - repeat - - How many times to repeat builds to check whether - they are deterministic. The default value is 0. If the value is - non-zero, every build is repeated the specified number of - times. If the contents of any of the runs differs from the - previous ones and is - true, the build is rejected and the resulting store paths are not - registered as “valid” in Nix’s database. - - - require-sigs - - If set to true (the default), - any non-content-addressed path added or copied to the Nix store - (e.g. when substituting from a binary cache) must have a valid - signature, that is, be signed using one of the keys listed in - or - . Set to false - to disable signature checking. - - - - - restrict-eval - - - - If set to true, the Nix evaluator will - not allow access to any files outside of the Nix search path (as - set via the NIX_PATH environment variable or the - option), or to URIs outside of - . The default is - false. - - - - - - run-diff-hook - - - If true, enable the execution of . - - - - When using the Nix daemon, run-diff-hook must - be set in the nix.conf configuration file, - and cannot be passed at the command line. - - - - - sandbox - - If set to true, builds will be - performed in a sandboxed environment, i.e., - they’re isolated from the normal file system hierarchy and will - only see their dependencies in the Nix store, the temporary build - directory, private versions of /proc, - /dev, /dev/shm and - /dev/pts (on Linux), and the paths configured with the - sandbox-paths - option. This is useful to prevent undeclared dependencies - on files in directories such as /usr/bin. In - addition, on Linux, builds run in private PID, mount, network, IPC - and UTS namespaces to isolate them from other processes in the - system (except that fixed-output derivations do not run in private - network namespace to ensure they can access the network). - - Currently, sandboxing only work on Linux and macOS. The use - of a sandbox requires that Nix is run as root (so you should use - the “build users” - feature to perform the actual builds under different users - than root). - - If this option is set to relaxed, then - fixed-output derivations and derivations that have the - __noChroot attribute set to - true do not run in sandboxes. - - The default is true on Linux and - false on all other platforms. - - - - - - sandbox-dev-shm-size - - This option determines the maximum size of the - tmpfs filesystem mounted on - /dev/shm in Linux sandboxes. For the format, - see the description of the option of - tmpfs in - mount8. The - default is 50%. - - - - - - sandbox-paths - - A list of paths bind-mounted into Nix sandbox - environments. You can use the syntax - target=source - to mount a path in a different location in the sandbox; for - instance, /bin=/nix-bin will mount the path - /nix-bin as /bin inside the - sandbox. If source is followed by - ?, then it is not an error if - source does not exist; for example, - /dev/nvidiactl? specifies that - /dev/nvidiactl will only be mounted in the - sandbox if it exists in the host filesystem. - - Depending on how Nix was built, the default value for this option - may be empty or provide /bin/sh as a - bind-mount of bash. - - - - - secret-key-files - - A whitespace-separated list of files containing - secret (private) keys. These are used to sign locally-built - paths. They can be generated using nix-store - --generate-binary-cache-key. The corresponding public - key can be distributed to other users, who can add it to - in their - nix.conf. - - - - - show-trace - - Causes Nix to print out a stack trace in case of Nix - expression evaluation errors. - - - - - substitute - - If set to true (default), Nix - will use binary substitutes if available. This option can be - disabled to force building from source. - - - - stalled-download-timeout - - The timeout (in seconds) for receiving data from servers - during download. Nix cancels idle downloads after this timeout's - duration. - - - - substituters - - A list of URLs of substituters, separated by - whitespace. The default is - https://cache.nixos.org. - - - - system - - This option specifies the canonical Nix system - name of the current installation, such as - i686-linux or - x86_64-darwin. Nix can only build derivations - whose system attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only - makes sense if the Nix binaries can run on multiple platforms, - e.g., ‘universal binaries’ that run on x86_64-linux and - i686-linux. - - It defaults to the canonical Nix system name detected by - configure at build time. - - - - - system-features - - A set of system “features” supported by this - machine, e.g. kvm. Derivations can express a - dependency on such features through the derivation attribute - requiredSystemFeatures. For example, the - attribute - - -requiredSystemFeatures = [ "kvm" ]; - - - ensures that the derivation can only be built on a machine with - the kvm feature. - - This setting by default includes kvm if - /dev/kvm is accessible, and the - pseudo-features nixos-test, - benchmark and big-parallel - that are used in Nixpkgs to route builds to specific - machines. - - - - - - tarball-ttl - - - Default: 3600 seconds. - - The number of seconds a downloaded tarball is considered - fresh. If the cached tarball is stale, Nix will check whether - it is still up to date using the ETag header. Nix will download - a new version if the ETag header is unsupported, or the - cached ETag doesn't match. - - - Setting the TTL to 0 forces Nix to always - check if the tarball is up to date. - - Nix caches tarballs in - $XDG_CACHE_HOME/nix/tarballs. - - Files fetched via NIX_PATH, - fetchGit, fetchMercurial, - fetchTarball, and fetchurl - respect this TTL. - - - - - timeout - - - - This option defines the maximum number of seconds that a - builder can run. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite loop - but keep writing to their standard output or standard error. It - can be overridden using the command line - switch. - - The value 0 means that there is no - timeout. This is also the default. - - - - - - trace-function-calls - - - - Default: false. - - If set to true, the Nix evaluator will - trace every function call. Nix will print a log message at the - "vomit" level for every function entrance and function exit. - - -function-trace entered undefined position at 1565795816999559622 -function-trace exited undefined position at 1565795816999581277 -function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 -function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - - The undefined position means the function - call is a builtin. - - Use the contrib/stack-collapse.py script - distributed with the Nix source code to convert the trace logs - in to a format suitable for flamegraph.pl. - - - - - - trusted-public-keys - - A whitespace-separated list of public keys. When - paths are copied from another Nix store (such as a binary cache), - they must be signed with one of these keys. For example: - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=. - - - - trusted-substituters - - A list of URLs of substituters, separated by - whitespace. These are not used by default, but can be enabled by - users of the Nix daemon by specifying --option - substituters urls on the - command line. Unprivileged users are only allowed to pass a - subset of the URLs listed in substituters and - trusted-substituters. - - - - trusted-users - - - - A list of names of users (separated by whitespace) that - have additional rights when connecting to the Nix daemon, such - as the ability to specify additional binary caches, or to import - unsigned NARs. You can also specify groups by prefixing them - with @; for instance, - @wheel means all users in the - wheel group. The default is - root. - - Adding a user to - is essentially equivalent to giving that user root access to the - system. For example, the user can set - and thereby obtain read access to - directories that are otherwise inacessible to - them. - - - - - - - - - - Deprecated Settings - - - - - - - binary-caches - - Deprecated: - binary-caches is now an alias to - . - - - - binary-cache-public-keys - - Deprecated: - binary-cache-public-keys is now an alias to - . - - - - build-compress-log - - Deprecated: - build-compress-log is now an alias to - . - - - - build-cores - - Deprecated: - build-cores is now an alias to - . - - - - build-extra-chroot-dirs - - Deprecated: - build-extra-chroot-dirs is now an alias to - . - - - - build-extra-sandbox-paths - - Deprecated: - build-extra-sandbox-paths is now an alias to - . - - - - build-fallback - - Deprecated: - build-fallback is now an alias to - . - - - - build-max-jobs - - Deprecated: - build-max-jobs is now an alias to - . - - - - build-max-log-size - - Deprecated: - build-max-log-size is now an alias to - . - - - - build-max-silent-time - - Deprecated: - build-max-silent-time is now an alias to - . - - - - build-repeat - - Deprecated: - build-repeat is now an alias to - . - - - - build-timeout - - Deprecated: - build-timeout is now an alias to - . - - - - build-use-chroot - - Deprecated: - build-use-chroot is now an alias to - . - - - - build-use-sandbox - - Deprecated: - build-use-sandbox is now an alias to - . - - - - build-use-substitutes - - Deprecated: - build-use-substitutes is now an alias to - . - - - - gc-keep-derivations - - Deprecated: - gc-keep-derivations is now an alias to - . - - - - gc-keep-outputs - - Deprecated: - gc-keep-outputs is now an alias to - . - - - - env-keep-derivations - - Deprecated: - env-keep-derivations is now an alias to - . - - - - extra-binary-caches - - Deprecated: - extra-binary-caches is now an alias to - . - - - - trusted-binary-caches - - Deprecated: - trusted-binary-caches is now an alias to - . - - - - - - - - From 7f60f48e1aed14680eaf05154abf243ea764bf85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= Date: Wed, 21 Oct 2020 17:04:36 +0200 Subject: [PATCH 087/227] Fix the docs about the new NIX_CONFIG env var This was accidentally documented as NIX_OPTIONS. --- doc/manual/src/command-ref/conf-file-prefix.md | 2 +- doc/manual/src/command-ref/env-common.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index d38456788..e53a3ddf4 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -19,7 +19,7 @@ By default Nix reads settings from the following places: and `XDG_CONFIG_HOME`. If these are unset, it will look in `$HOME/.config/nix.conf`. - - If `NIX_OPTIONS` is set, its contents is treated as the contents of + - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. The configuration files consist of `name = diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 27e730fc8..c670d82b8 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -81,7 +81,7 @@ Most Nix commands interpret the following environment variables: Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). - - `NIX_OPTIONS` + - `NIX_CONFIG` Applies settings from Nix configuration from the environment. The content is treated as if it was read from a Nix configuration file. Settings are separated by the newline character. From e556a1beb73f5d5608d7a8bad0a0c13fd148ed0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Oct 2020 17:54:21 +0200 Subject: [PATCH 088/227] nix develop: Handle 'declare -ax' in bash output Fixes 'nix develop nixpkgs#qpdfview'. --- src/nix/develop.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 380417c82..32669414b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -39,21 +39,24 @@ BuildEnvironment readEnvironment(const Path & path) static std::string varNameRegex = R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; - static std::regex declareRegex( - "^declare -x (" + varNameRegex + ")" + - R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); - static std::string simpleStringRegex = R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; - static std::string quotedStringRegex = + static std::string dquotedStringRegex = + R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + + static std::string squotedStringRegex = R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; static std::string indexedArrayRegex = R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; + static std::regex declareRegex( + "^declare -a?x (" + varNameRegex + ")(=(" + + dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); + static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); /* Note: we distinguish between an indexed and associative array using the space before the closing parenthesis. Will From f9438fb64a223c05ebcfffa9706e1ca811a87d70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:06:36 +0200 Subject: [PATCH 089/227] nix develop: Add --redirect flag to redirect dependencies This is primarily useful if you're hacking simultaneously on a package and one of its dependencies. E.g. if you're hacking on Hydra and Nix, you would start a dev shell for Nix, and then a dev shell for Hydra as follows: $ nix develop \ --redirect .#hydraJobs.build.x86_64-linux.nix ~/Dev/nix/outputs/out \ --redirect .#hydraJobs.build.x86_64-linux.nix.dev ~/Dev/nix/outputs/dev (This assumes hydraJobs.build.x86_64-linux has a passthru.nix attribute. You can also use a store path.) This causes all references in the environment to those store paths to be rewritten to ~/Dev/nix/outputs/{out,dev}. Note: unfortunately, you may need to set LD_LIBRARY_PATH=~/Dev/nix/outputs/out/lib because Nixpkgs' ld-wrapper only adds -rpath entries for -L flags that point to the Nix store. --- src/nix/develop.cc | 53 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 32669414b..7a5f7e218 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -185,7 +185,22 @@ struct Common : InstallableCommand, MixProfile "UID", }; + std::vector> redirects; + + Common() + { + addFlag({ + .longName = "redirect", + .description = "redirect a store path to a mutable location", + .labels = {"installable", "outputs-dir"}, + .handler = {[&](std::string installable, std::string outputsDir) { + redirects.push_back({installable, outputsDir}); + }} + }); + } + std::string makeRcScript( + ref store, const BuildEnvironment & buildEnvironment, const Path & outputsDir = absPath(".") + "/outputs") { @@ -217,6 +232,8 @@ struct Common : InstallableCommand, MixProfile out << "eval \"$shellHook\"\n"; + auto script = out.str(); + /* Substitute occurrences of output paths. */ auto outputs = buildEnvironment.env.find("outputs"); assert(outputs != buildEnvironment.env.end()); @@ -230,7 +247,33 @@ struct Common : InstallableCommand, MixProfile rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); } - return rewriteStrings(out.str(), rewrites); + /* Substitute redirects. */ + for (auto & [installableS, dir] : redirects) { + dir = absPath(dir); + auto installable = parseInstallable(store, installableS); + auto buildable = installable->toBuildable(); + auto doRedirect = [&](const StorePath & path) + { + auto from = store->printStorePath(path); + if (script.find(from) == std::string::npos) + warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); + else { + printInfo("redirecting '%s' to '%s'", from, dir); + rewrites.insert({from, dir}); + } + }; + std::visit(overloaded { + [&](const BuildableOpaque & bo) { + doRedirect(bo.path); + }, + [&](const BuildableFromDrv & bfd) { + for (auto & [outputName, path] : bfd.outputs) + if (path) doRedirect(*path); + }, + }, buildable); + } + + return rewriteStrings(script, rewrites); } Strings getDefaultFlakeAttrPaths() override @@ -348,6 +391,10 @@ struct CmdDevelop : Common, MixEnvironment "To use a build environment previously recorded in a profile:", "nix develop /tmp/my-shell" }, + Example{ + "To replace all occurences of a store path with a writable directory:", + "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev" + }, }; } @@ -357,7 +404,7 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(buildEnvironment); + auto script = makeRcScript(store, buildEnvironment); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -449,7 +496,7 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(buildEnvironment); + std::cout << makeRcScript(store, buildEnvironment); } }; From 750ce500c221ecd4720a5b02e3f3cbb0bc05ef9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Oct 2020 12:03:15 +0200 Subject: [PATCH 090/227] Fix clang build --- src/nix/develop.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7a5f7e218..d3c4761a7 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -248,9 +248,9 @@ struct Common : InstallableCommand, MixProfile } /* Substitute redirects. */ - for (auto & [installableS, dir] : redirects) { - dir = absPath(dir); - auto installable = parseInstallable(store, installableS); + for (auto & [installable_, dir_] : redirects) { + auto dir = absPath(dir_); + auto installable = parseInstallable(store, installable_); auto buildable = installable->toBuildable(); auto doRedirect = [&](const StorePath & path) { From c189cf7e3361e3c4caa989bde057439485b2c227 Mon Sep 17 00:00:00 2001 From: tnias Date: Sun, 25 Oct 2020 23:16:53 +0100 Subject: [PATCH 091/227] Add sha512 to hashAlgo listings in manpages (#4186) --- doc/manual/src/command-ref/nix-hash.md | 2 +- doc/manual/src/command-ref/nix-prefetch-url.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 7ed82cdfc..de0459b9e 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -45,7 +45,7 @@ md5sum`. - `--type` *hashAlgo* Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, and `sha256`. + `md5`, `sha1`, `sha256`, and `sha512`. - `--to-base16` Don’t hash anything, but convert the base-32 hash representation diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 78c612cd4..59ab89b29 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,7 +39,7 @@ Nix store is also printed. - `--type` *hashAlgo* Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, and `sha256`. + `md5`, `sha1`, `sha256`, and `sha512`. - `--print-path` Print the store path of the downloaded file on standard output. From ac0e24f21b162f99c8e9437c710797820345a7a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 11:28:42 +0100 Subject: [PATCH 092/227] Revert "Bump version to 3.0" This reverts commit 189e6f5e1d949f50ab0b6e5acd25e230d206692d. After some discussion, it seems better not to bump the major version number since most of the new features since 2.3 are marked experimental. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f398a2061..7208c2182 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.0 \ No newline at end of file +2.4 \ No newline at end of file From dc7d1322efbaa176bff38b1ad15eab6e11c83340 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 14:24:25 +0100 Subject: [PATCH 093/227] Make the prompt used in development shells configurable --- src/nix/develop.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d3c4761a7..8fea7ee9c 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -11,6 +11,19 @@ using namespace nix; +struct DevelopSettings : Config +{ + Setting bashPrompt{this, "", "bash-prompt", + "The bash prompt (`PS1`) in `nix develop` shells."}; + + Setting bashPromptSuffix{this, "", "bash-prompt-suffix", + "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; +}; + +static DevelopSettings developSettings; + +static GlobalConfig::Register rDevelopSettings(&developSettings); + struct Var { bool exported = true; @@ -429,6 +442,10 @@ struct CmdDevelop : Common, MixEnvironment else { script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + if (developSettings.bashPrompt != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); + if (developSettings.bashPromptSuffix != "") + script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix)); } writeFull(rcFileFd.get(), script); From 9d5e9ef0da89fe4fd02d7053ee28d79df3245325 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 16:58:58 +0100 Subject: [PATCH 094/227] Move Explicit --- src/libexpr/flake/lockfile.cc | 3 ++- src/libexpr/primops/fetchTree.cc | 5 +++-- src/libfetchers/attrs.hh | 12 ------------ src/libutil/types.hh | 12 ++++++++++++ src/nix/flake.cc | 6 +++--- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index bb46e1bb4..a01a63611 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -34,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isImmutable()) - throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); + throw Error("lockfile contains mutable lock '%s'", + fetchers::attrsToJson(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 7cd4d0fbf..8d7ae4c14 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -43,7 +43,8 @@ void emitTreeAttrs( } if (input.getType() == "git") - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), + fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); @@ -101,7 +102,7 @@ static void fetchTree( else if (attr.value->type == tString) addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); + attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type == tInt) attrs.emplace(attr.name, attr.value->integer); else diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 4b4630c80..56bcdcfc8 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -8,18 +8,6 @@ namespace nix::fetchers { -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ -template -struct Explicit { - T t; - - bool operator ==(const Explicit & other) const - { - return t == other.t; - } -}; - typedef std::variant> Attr; typedef std::map Attrs; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6c4c5ab74..9c85fef62 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -34,4 +34,16 @@ struct OnStartup OnStartup(T && t) { t(); } }; +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; + + bool operator ==(const Explicit & other) const + { + return t == other.t; + } +}; + } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 43176d887..790e1ce95 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -82,11 +82,11 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake) if (flake.description) j["description"] = *flake.description; j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = attrsToJson(flake.originalRef.toAttrs()); + j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs()); j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); + j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); + j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); if (auto revCount = flake.lockedRef.input.getRevCount()) From 1e66d146a343cff8ea60fa24b1642762de27c787 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 17:59:32 +0100 Subject: [PATCH 095/227] Fix test --- tests/binary-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index e14cf882e..6415c9302 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -271,5 +271,5 @@ rm $cacheDir/$(hashpart $docPath).narinfo nix-store --delete $outPath $docPath # -vvv is the level that logs during the loop -timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ +timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From b875b8f45c8d73c26e2cf13843fa25cc6762eebc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 17:59:36 +0100 Subject: [PATCH 096/227] Remove edition field --- src/libexpr/flake/flake.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bae4d65e5..ca3b185f9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -196,11 +196,6 @@ static Flake getFlake( expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); - auto sEdition = state.symbols.create("edition"); // FIXME: remove soon - - if (vInfo.attrs->get(sEdition)) - warn("flake '%s' has deprecated attribute 'edition'", lockedRef); - if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, tString, *description->value, *description->pos); flake.description = description->value->string.s; @@ -229,8 +224,7 @@ static Flake getFlake( throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); for (auto & attr : *vInfo.attrs) { - if (attr.name != sEdition && - attr.name != state.sDescription && + if (attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs) throw Error("flake '%s' has an unsupported attribute '%s', at %s", From 14aecbb288b71bbb79ddca638918318c9298200b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:36:46 +0100 Subject: [PATCH 097/227] BaseSetting::set(): Don't append to previous value --- src/libutil/config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 521733025..eef01bde2 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -270,6 +270,7 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str) { + value.clear(); auto kvpairs = tokenizeString(str); for (auto & s : kvpairs) { From 731edf0d9be27d1a64c9645595c7efba31dde2b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:37:11 +0100 Subject: [PATCH 098/227] isTrivial(): Support trivial lists --- src/libexpr/eval.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6366050c..4de87d647 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -208,7 +208,8 @@ bool Value::isTrivial() const && (type != tThunk || (dynamic_cast(thunk.expr) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) - || dynamic_cast(thunk.expr)); + || dynamic_cast(thunk.expr) + || dynamic_cast(thunk.expr)); } From 343239fc8a1993f707a990c2cd54a41f1fa3de99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:45:39 +0100 Subject: [PATCH 099/227] Allow nix.conf options to be set in flake.nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it possible to have per-project configuration in flake.nix, e.g. binary caches and other stuff: nixConfig.bash-prompt-suffix = "ngi# "; nixConfig.substituters = [ "https://cache.ngi0.nixos.org/" ]; --- src/libexpr/flake/flake.cc | 70 +++++++++++++++++++++++++++++++++++--- src/libexpr/flake/flake.hh | 11 +++++- src/nix/installables.cc | 5 ++- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ca3b185f9..bdcf63c21 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -71,11 +71,17 @@ static std::tuple fetchOrSubstituteTree( return {std::move(tree), resolvedRef, lockedRef}; } -static void expectType(EvalState & state, ValueType type, - Value & value, const Pos & pos) +static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) { if (value.type == tThunk && value.isTrivial()) state.forceValue(value, pos); +} + + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + forceTrivialValue(state, value, pos); if (value.type != type) throw Error("expected %s but got %s at %s", showType(type), showType(value.type), pos); @@ -114,7 +120,6 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, tString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - state.forceValue(*attr.value); if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else @@ -223,10 +228,41 @@ static Flake getFlake( } else throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + auto sNixConfig = state.symbols.create("nixConfig"); + + if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { + expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); + + for (auto & option : *nixConfig->value->attrs) { + forceTrivialValue(state, *option.value, *option.pos); + if (option.value->type == tString) + flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)}); + else if (option.value->type == tInt) + flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)}); + else if (option.value->type == tBool) + flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)}); + else if (option.value->isList()) { + std::vector ss; + for (unsigned int n = 0; n < option.value->listSize(); ++n) { + auto elem = option.value->listElems()[n]; + if (elem->type != tString) + throw TypeError("list element in flake configuration option '%s' is %s while a string is expected", + option.name, showType(*option.value)); + ss.push_back(state.forceStringNoCtx(*elem, *option.pos)); + } + flake.config.options.insert({option.name, ss}); + } + else + throw TypeError("flake configuration option '%s' is %s", + option.name, showType(*option.value)); + } + } + for (auto & attr : *vInfo.attrs) { if (attr.name != state.sDescription && attr.name != sInputs && - attr.name != sOutputs) + attr.name != sOutputs && + attr.name != sNixConfig) throw Error("flake '%s' has an unsupported attribute '%s', at %s", lockedRef, attr.name, *attr.pos); } @@ -599,4 +635,30 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } +void ConfigFile::apply() +{ + for (auto & [name, value] : options) { + // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU. + if (name != "bash-prompt-suffix" && + name != "bash-prompt" && + name != "substituters" && + name != "extra-substituters") + { + warn("ignoring untrusted flake configuration option '%s'", name); + continue; + } + // FIXME: Move into libutil/config.cc. + if (auto s = std::get_if(&value)) + globalConfig.set(name, *s); + else if (auto n = std::get_if(&value)) + globalConfig.set(name, fmt("%d", n)); + else if (auto b = std::get_if>(&value)) + globalConfig.set(name, b->t ? "true" : "false"); + else if (auto ss = std::get_if>(&value)) + globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil + else + assert(false); + } +} + } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index cf62c7741..7eebd9044 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -47,8 +47,16 @@ struct FlakeInput FlakeInputs overrides; }; -// The Flake structure is the main internal representation of a flake.nix file. +struct ConfigFile +{ + using ConfigValue = std::variant, std::vector>; + std::map options; + + void apply(); +}; + +/* The contents of a flake.nix file. */ struct Flake { FlakeRef originalRef; // the original flake specification (by the user) @@ -57,6 +65,7 @@ struct Flake std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; + ConfigFile config; // 'nixConfig' attribute ~Flake(); }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 7473c9758..fb264491a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -533,8 +533,11 @@ InstallableFlake::getCursors(EvalState & state) std::shared_ptr InstallableFlake::getLockedFlake() const { - if (!_lockedFlake) + if (!_lockedFlake) { _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); + _lockedFlake->flake.config.apply(); + // FIXME: send new config to the daemon. + } return _lockedFlake; } From c092fa4702215fdb61611c5dd28194401d056170 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Sep 2020 16:30:42 +0200 Subject: [PATCH 100/227] Allow non-CA derivations to depend on CA derivations --- src/libexpr/primops.cc | 39 +++++++++++----- src/libstore/build/derivation-goal.cc | 12 ++++- src/libstore/derivations.cc | 65 ++++++++++++++++++++++++--- src/libstore/derivations.hh | 14 +++++- src/libstore/local-store.cc | 4 +- src/nix/show-derivation.cc | 1 + tests/content-addressed.nix | 11 ++++- tests/content-addressed.sh | 1 + 8 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..236433ef1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1089,18 +1089,35 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // Regular, non-CA derivation should always return a single hash and not // hash per output. - Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true)); + auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + std::visit(overloaded { + [&](Hash h) { + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }); + } + }, + [&](CaOutputHashes) { + // Shouldn't happen as the toplevel derivation is not CA. + assert(false); + }, + [&](UnknownHashes) { + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputDeferred{}, + }); + } + }, + }, + hashModulo); - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, - }); - } } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index db0c2bb6c..cc8737fd5 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,7 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + if ((!fullDrv.inputDrvs.empty() && + fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3166,6 +3167,15 @@ void DerivationGoal::registerOutputs() [&](DerivationOutputCAFloating dof) { return newInfoFromCA(dof); }, + [&](DerivationOutputDeferred) { + // No derivation should reach that point without having been + // rewritten first + assert(false); + // Ugly, but the compiler insists on having this return a value + // of type `ValidPathInfo` despite the `assert(false)`, so + // let's provide it + return *(ValidPathInfo*)0; + }, }, output.output); /* Calculate where we'll move the output files. In the checking case we diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..3b3a25391 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,6 +21,9 @@ std::optional DerivationOutput::path(const Store & store, std::string [](DerivationOutputCAFloating dof) -> std::optional { return std::nullopt; }, + [](DerivationOutputDeferred) -> std::optional { + return std::nullopt; + }, }, output); } @@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return true; + case DerivationType::DeferredInputAddressed: return false; }; // Since enums can have non-variant values, but making a `default:` would // disable exhaustiveness warnings. @@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store, }; } } else { + if (pathS == "") { + return DerivationOutput { + .output = DerivationOutputDeferred { } + }; + } validatePath(pathS); return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, + [&](DerivationOutputDeferred) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } }, i.second.output); s += ')'; } @@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { @@ -408,22 +424,27 @@ DerivationType BasicDerivation::type() const throw Error("All floating outputs must use the same hash type"); } }, + [&](DerivationOutputDeferred _) { + deferredIAOutputs.insert(i.first); + }, }, i.second.output); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::InputAddressed; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? throw Error("Only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") throw Error("Single fixed output must be named \"out\""); return DerivationType::CAFixed; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::CAFloating; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { + return DerivationType::DeferredInputAddressed; } else { throw Error("Can't mix derivation output types"); } @@ -454,6 +475,8 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } +UnknownHashes unknownHashes; + /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -476,7 +499,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); + return unknownHashes; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -491,12 +514,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } case DerivationType::InputAddressed: break; + case DerivationType::DeferredInputAddressed: + break; } /* For other derivations, replace the inputs paths with recursive calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { + bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation @@ -514,7 +540,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, + [&](UnknownHashes) { + hasUnknownHash = true; + }, }, res); + if (hasUnknownHash) { + return unknownHashes; + } } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); @@ -620,6 +652,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, + [&](DerivationOutputDeferred) { + out << "" + << "" + << ""; + }, }, i.second.output); } worker_proto::write(store, out, drv.inputSrcs); @@ -645,7 +682,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -// N.B. Outputs are left unchanged static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { debug("Rewriting the derivation"); @@ -666,6 +702,21 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String newEnv.emplace(envName, envValue); } drv.env = newEnv; + + auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); + for (auto & [outputName, output] : drv.outputs) { + if (std::holds_alternative(output.output)) { + Hash h = std::get(hashModulo); + auto outPath = store.makeOutputPath(outputName, h, drv.name); + drv.env[outputName] = store.printStorePath(outPath); + output = DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }; + } + } + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d292b2e5..f9ba935e6 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,12 +41,18 @@ struct DerivationOutputCAFloating HashType hashType; }; +/* Input-addressed output which depends on a (CA) derivation whose hash isn't + * known atm + */ +struct DerivationOutputDeferred {}; + struct DerivationOutput { std::variant< DerivationOutputInputAddressed, DerivationOutputCAFixed, - DerivationOutputCAFloating + DerivationOutputCAFloating, + DerivationOutputDeferred > output; std::optional hashAlgoOpt(const Store & store) const; /* Note, when you use this function you should make sure that you're passing @@ -72,6 +78,7 @@ typedef std::map StringPairs; enum struct DerivationType : uint8_t { InputAddressed, + DeferredInputAddressed, CAFixed, CAFloating, }; @@ -167,9 +174,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; +struct UnknownHashes {}; + typedef std::variant< Hash, // regular DRV normalized hash - CaOutputHashes + CaOutputHashes, // Fixed-output derivation hashes + UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29236a9c..d29a68179 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -573,6 +573,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](DerivationOutputCAFloating _) { /* Nothing to check */ }, + [&](DerivationOutputDeferred) { + }, }, i.second.output); } } @@ -817,7 +819,7 @@ std::map> LocalStore::queryPartialDerivati } /* can't just use else-if instead of `!haveCached` because we need to unlock `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && drv.type() == DerivationType::CAFloating) { + if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { /* Try resolve drv and use that path instead. */ auto attempt = drv.tryResolve(*this); if (!attempt) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2542537d3..6d4f295d7 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -82,6 +82,7 @@ struct CmdShowDerivation : InstallablesCommand [&](DerivationOutputCAFloating dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, + [&](DerivationOutputDeferred) {}, }, output.output); } } diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 3dcf916c3..8ca96d4bf 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -15,7 +15,7 @@ rec { ''; }; rootCA = mkDerivation { - name = "dependent"; + name = "rootCA"; outputs = [ "out" "dev" ]; buildCommand = '' echo "building a CA derivation" @@ -51,4 +51,13 @@ rec { outputHashMode = "recursive"; outputHashAlgo = "sha256"; }; + dependentNonCA = mkDerivation { + name = "dependent-non-ca"; + buildCommand = '' + echo "Didn't cut-off" + echo "building dependent-non-ca" + mkdir -p $out + echo ${rootCA}/non-ca-hello > $out/dep + ''; + }; } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 61ec03fe3..547919660 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -22,6 +22,7 @@ secondSeedArgs=(-j0) # dependent derivations always being already built. #testDerivation dependentCA testDerivation transitivelyDependentCA +testDerivation dependentNonCA nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true From bc081bcd816542d66f1578788b93df4d7e07b135 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 24 Sep 2020 10:11:58 +0200 Subject: [PATCH 101/227] Inline `unkownHashes` See https://github.com/NixOS/nix/pull/4056#discussion_r493661632 --- src/libstore/derivations.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3b3a25391..517ecfaa2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -475,8 +475,6 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } -UnknownHashes unknownHashes; - /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -499,7 +497,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - return unknownHashes; + return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -545,7 +543,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m }, }, res); if (hasUnknownHash) { - return unknownHashes; + return UnknownHashes {}; } } From ab21ab65016275c224d1d40c42bdfed80dfbcbb0 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 7 Oct 2020 12:24:53 +0200 Subject: [PATCH 102/227] Test the remote caching of non-ca-depending-on-ca derivations Although the non-resolved derivation will never get a cache-hit (it doesn't have an output path to query the cache for anyways), we might get one on the resolved derivation. --- tests/content-addressed.sh | 55 +++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 547919660..bdab09c86 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -5,24 +5,49 @@ source common.sh drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1) nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 -testDerivation () { +buildAttr () { local derivationPath=$1 - local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + shift + local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +testRemoteCache () { + clearCache + local outPath=$(buildAttr dependentNonCA) + nix copy --to file://$cacheDir $outPath + clearStore + buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") +} + +testDeterministicCA () { + [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]] +} + +testCutoffFor () { local out1 out2 - out1=$(nix-build "${commonArgs[@]}" --arg seed 1) - out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${secondSeedArgs[@]}") + out1=$(buildAttr $1) + # The seed only changes the root derivation, and not it's output, so the + # dependent derivations should only need to be built once. + out2=$(buildAttr $1 -j0) test "$out1" == "$out2" } -testDerivation rootCA -# The seed only changes the root derivation, and not it's output, so the -# dependent derivations should only need to be built once. -secondSeedArgs=(-j0) -# Don't directly build depenentCA, that way we'll make sure we dodn't rely on -# dependent derivations always being already built. -#testDerivation dependentCA -testDerivation transitivelyDependentCA -testDerivation dependentNonCA +testCutoff () { + # Don't directly build depenentCA, that way we'll make sure we dodn't rely on + # dependent derivations always being already built. + #testDerivation dependentCA + testCutoffFor transitivelyDependentCA + testCutoffFor dependentNonCA +} -nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 -nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true +testGC () { + nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 + nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true +} + +testRemoteCache +testDeterministicCA +testCutoff +testGC From 82e4d2a82ef11a95ff8e9aba3062db50145daab1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 28 Oct 2020 05:13:18 +0000 Subject: [PATCH 103/227] No x86_32 static nix jobs for now Fixes #4175 --- flake.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0233962b4..2abbdff53 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,8 @@ officialRelease = false; - linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; + linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; + linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); @@ -228,7 +229,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); + buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -451,7 +452,7 @@ packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; - } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { + } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { From a5019f0508be961bf0230d2a528d30d3ded4b12a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Oct 2020 20:45:57 +0100 Subject: [PATCH 104/227] Consistency --- src/libutil/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 4a94f0dfd..8efff190a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -192,7 +192,7 @@ Hash Hash::parseAny(std::string_view original, std::optional optType) // Either the string or user must provide the type, if they both do they // must agree. if (!optParsedType && !optType) - throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); else if (optParsedType && optType && *optParsedType != *optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); From 6a4bf535d8126d8dc7306b3940bb49e0cc014a56 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 28 Oct 2020 09:41:18 -0500 Subject: [PATCH 105/227] Capitalize JSON for consistency --- src/libexpr/flake/lockfile.cc | 14 +++++++------- src/libexpr/flake/lockfile.hh | 2 +- src/libfetchers/attrs.cc | 4 ++-- src/libfetchers/attrs.hh | 2 +- src/libfetchers/cache.cc | 20 ++++++++++---------- src/libfetchers/fetchers.cc | 6 +++--- src/libfetchers/registry.cc | 6 +++--- src/libmain/loggers.cc | 4 ++-- src/libmain/loggers.hh | 2 +- src/libutil/logging.cc | 2 +- src/nix/flake.cc | 12 ++++++------ 11 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a01a63611..8e2f7131f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json) { if (!lockedRef.input.isImmutable()) throw Error("lockfile contains mutable lock '%s'", - fetchers::attrsToJson(lockedRef.input.toAttrs())); + fetchers::attrsToJSON(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const @@ -111,7 +111,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) // a bit since we don't need to worry about cycles. } -nlohmann::json LockFile::toJson() const +nlohmann::json LockFile::toJSON() const { nlohmann::json nodes; std::unordered_map, std::string> nodeKeys; @@ -155,8 +155,8 @@ nlohmann::json LockFile::toJson() const } if (auto lockedNode = std::dynamic_pointer_cast(node)) { - n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); - n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); if (!lockedNode->isFlake) n["flake"] = false; } @@ -175,7 +175,7 @@ nlohmann::json LockFile::toJson() const std::string LockFile::to_string() const { - return toJson().dump(2); + return toJSON().dump(2); } LockFile LockFile::read(const Path & path) @@ -186,7 +186,7 @@ LockFile LockFile::read(const Path & path) std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - stream << lockFile.toJson().dump(2); + stream << lockFile.toJSON().dump(2); return stream; } @@ -224,7 +224,7 @@ bool LockFile::isImmutable() const bool LockFile::operator ==(const LockFile & other) const { // FIXME: slow - return toJson() == other.toJson(); + return toJSON() == other.toJSON(); } InputPath parseInputPath(std::string_view s) diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 627794d8c..96f1edc76 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -52,7 +52,7 @@ struct LockFile LockFile() {}; LockFile(const nlohmann::json & json, const Path & path); - nlohmann::json toJson() const; + nlohmann::json toJSON() const; std::string to_string() const; diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 1e59faa73..720b19fcd 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) return attrs; } -nlohmann::json attrsToJson(const Attrs & attrs) +nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { @@ -44,7 +44,7 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin if (i == attrs.end()) return {}; if (auto v = std::get_if(&i->second)) return *v; - throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); + throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } std::string getStrAttr(const Attrs & attrs, const std::string & name) diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 56bcdcfc8..a2d53a7bf 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,7 +13,7 @@ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); -nlohmann::json attrsToJson(const Attrs & attrs); +nlohmann::json attrsToJSON(const Attrs & attrs); std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index e1c7f3dee..34ff6f85b 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -55,8 +55,8 @@ struct CacheImpl : Cache bool immutable) override { _state.lock()->add.use() - (attrsToJson(inAttrs).dump()) - (attrsToJson(infoAttrs).dump()) + (attrsToJSON(inAttrs).dump()) + (attrsToJSON(infoAttrs).dump()) (store->printStorePath(storePath)) (immutable) (time(0)).exec(); @@ -70,7 +70,7 @@ struct CacheImpl : Cache if (!res->expired) return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); debug("ignoring expired cache entry '%s'", - attrsToJson(inAttrs).dump()); + attrsToJSON(inAttrs).dump()); } return {}; } @@ -81,15 +81,15 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto inAttrsJson = attrsToJson(inAttrs).dump(); + auto inAttrsJSON = attrsToJSON(inAttrs).dump(); - auto stmt(state->lookup.use()(inAttrsJson)); + auto stmt(state->lookup.use()(inAttrsJSON)); if (!stmt.next()) { - debug("did not find cache entry for '%s'", inAttrsJson); + debug("did not find cache entry for '%s'", inAttrsJSON); return {}; } - auto infoJson = stmt.getStr(0); + auto infoJSON = stmt.getStr(0); auto storePath = store->parseStorePath(stmt.getStr(1)); auto immutable = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); @@ -97,16 +97,16 @@ struct CacheImpl : Cache store->addTempRoot(storePath); if (!store->isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. - debug("ignoring disappeared cache entry '%s'", inAttrsJson); + debug("ignoring disappeared cache entry '%s'", inAttrsJSON); return {}; } debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJson, infoJson, store->printStorePath(storePath)); + inAttrsJSON, infoJSON, store->printStorePath(storePath)); return Result { .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), - .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), .storePath = std::move(storePath) }; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..e6741a451 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -65,7 +65,7 @@ Input Input::fromAttrs(Attrs && attrs) ParsedURL Input::toURL() const { if (!scheme) - throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs)); return scheme->toURL(*this); } @@ -110,7 +110,7 @@ bool Input::contains(const Input & other) const std::pair Input::fetch(ref store) const { if (!scheme) - throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the @@ -247,7 +247,7 @@ std::optional Input::getLastModified() const ParsedURL InputScheme::toURL(const Input & input) { - throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); + throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } Input InputScheme::applyOverrides( diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 2426882ca..81b2227de 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -60,10 +60,10 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from.toAttrs()); - obj["to"] = attrsToJson(entry.to.toAttrs()); + obj["from"] = attrsToJSON(entry.from.toAttrs()); + obj["to"] = attrsToJSON(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) - obj["to"].update(attrsToJson(entry.extraAttrs)); + obj["to"].update(attrsToJSON(entry.extraAttrs)); if (entry.exact) obj["exact"] = true; arr.emplace_back(std::move(obj)); diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 0a7291780..cdf23859b 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) { else if (logFormatStr == "raw-with-logs") return LogFormat::rawWithLogs; else if (logFormatStr == "internal-json") - return LogFormat::internalJson; + return LogFormat::internalJSON; else if (logFormatStr == "bar") return LogFormat::bar; else if (logFormatStr == "bar-with-logs") @@ -26,7 +26,7 @@ Logger * makeDefaultLogger() { return makeSimpleLogger(false); case LogFormat::rawWithLogs: return makeSimpleLogger(true); - case LogFormat::internalJson: + case LogFormat::internalJSON: return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index cada03110..f3c759193 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -7,7 +7,7 @@ namespace nix { enum class LogFormat { raw, rawWithLogs, - internalJson, + internalJSON, bar, barWithLogs, }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8a6752e22..6fd0dacef 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -306,7 +306,7 @@ bool handleJSONLogMessage(const std::string & msg, } catch (std::exception & e) { logError({ - .name = "Json log message", + .name = "JSON log message", .hint = hintfmt("bad log message from builder: %s", e.what()) }); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 790e1ce95..7a7c71676 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -76,17 +76,17 @@ static void printFlakeInfo(const Store & store, const Flake & flake) std::put_time(std::localtime(&*lastModified), "%F %T")); } -static nlohmann::json flakeToJson(const Store & store, const Flake & flake) +static nlohmann::json flakeToJSON(const Store & store, const Flake & flake) { nlohmann::json j; if (flake.description) j["description"] = *flake.description; j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs()); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs()); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs()); + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -139,7 +139,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON auto flake = getFlake(); if (json) { - auto json = flakeToJson(*store, flake); + auto json = flakeToJSON(*store, flake); logger->cout("%s", json.dump()); } else printFlakeInfo(*store, flake); @@ -158,7 +158,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON auto flake = lockFlake(); if (json) - logger->cout("%s", flake.lockFile.toJson()); + logger->cout("%s", flake.lockFile.toJSON()); else { logger->cout("%s", flake.flake.lockedRef); From 869c0321ff4829f56e10316f401a0f9268c1a8b0 Mon Sep 17 00:00:00 2001 From: stev Date: Thu, 29 Oct 2020 00:33:14 +0100 Subject: [PATCH 106/227] Alter "wanted:" to "specified:" in hash mismatch output This makes it even clearer which of the two hashes was specified in the nix files. Some may think that "wanted" and "got" is obvious, but: "got" could mean "got in nix file" and "wanted" could mean "want to see in nix file". --- src/libexpr/primops/fetchTree.cc | 2 +- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/local-store.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8d7ae4c14..e6f637a43 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? state.store->queryPathInfo(storePath)->narHash : hashFile(htSHA256, path); if (hash != *expectedHash) - throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index cc8737fd5..7db83c8be 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3157,7 +3157,7 @@ void DerivationGoal::registerOutputs() valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), wanted.to_string(SRI, true), got.to_string(SRI, true))); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29a68179..bfad8fb21 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1085,11 +1085,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); autoGC(); From ff4dea63c9403880500f82ce273713ecf793d2d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Oct 2020 18:17:39 +0100 Subject: [PATCH 107/227] Generalize extra-* settings This removes the extra-substituters and extra-sandbox-paths settings and instead makes every array setting extensible by setting "extra- = " in the configuration file or passing "-- " on the command line. --- .../src/command-ref/conf-file-prefix.md | 27 ++++++--- src/libstore/build/derivation-goal.cc | 6 +- src/libstore/daemon.cc | 2 - src/libstore/globals.cc | 9 ++- src/libstore/globals.hh | 21 +------ src/libstore/store-api.cc | 3 - src/libutil/config.cc | 58 +++++++++++++++---- src/libutil/config.hh | 11 +++- 8 files changed, 83 insertions(+), 54 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index e53a3ddf4..3140170ab 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -22,19 +22,30 @@ By default Nix reads settings from the following places: - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. -The configuration files consist of `name = -value` pairs, one per line. Other files can be included with a line like -`include -path`, where *path* is interpreted relative to the current conf file and -a missing file is an error unless `!include` is used instead. Comments +The configuration files consist of `name = value` pairs, one per +line. Other files can be included with a line like `include path`, +where *path* is interpreted relative to the current conf file and a +missing file is an error unless `!include` is used instead. Comments start with a `#` character. Here is an example configuration file: keep-outputs = true # Nice for developers keep-derivations = true # Idem -You can override settings on the command line using the `--option` flag, -e.g. `--option keep-outputs -false`. +You can override settings on the command line using the `--option` +flag, e.g. `--option keep-outputs false`. Every configuration setting +also has a corresponding command line flag, e.g. `--max-jobs 16`; for +Boolean settings, there are two flags to enable or disable the setting +(e.g. `--keep-failed` and `--no-keep-failed`). + +A configuration setting usually overrides any previous value. However, +you can prefix the name of the setting by `extra-` to *append* to the +previous value. For instance, + + substituters = a b + extra-substituters = c d + +defines the `substituters` setting to be `a b c d`. This is also +available as a command line flag (e.g. `--extra-substituters`). The following settings are currently available: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7db83c8be..3dacb218c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1332,13 +1332,9 @@ void DerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - dirsInChroot.clear(); - for (auto i : dirs) { + for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; bool optional = false; if (i[i.size() - 1] == '?') { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 99d8add92..4dbc7ba38 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -231,8 +231,6 @@ struct ClientSettings settings.set(name, value); else if (setSubstituters(settings.substituters)) ; - else if (setSubstituters(settings.extraSubstituters)) - ; else debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4df68d0c9..f38601d6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -160,7 +160,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { if (str == "true") value = smEnabled; else if (str == "relaxed") value = smRelaxed; @@ -168,6 +168,11 @@ template<> void BaseSetting::set(const std::string & str) else throw UsageError("option '%s' has invalid value '%s'", name, str); } +template<> bool BaseSetting::isAppendable() +{ + return false; +} + template<> std::string BaseSetting::to_string() const { if (value == smEnabled) return "true"; @@ -198,7 +203,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str) +void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); else if (!string2Int(str, value)) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8c63c5b34..eabd83e3f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -25,7 +25,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; }; class Settings : public Config { @@ -413,14 +413,6 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting extraSandboxPaths{ - this, {}, "extra-sandbox-paths", - R"( - A list of additional paths appended to `sandbox-paths`. Useful if - you want to extend its default value. - )", - {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting buildRepeat{ this, 0, "repeat", R"( @@ -599,17 +591,6 @@ public: )", {"binary-caches"}}; - // FIXME: provide a way to add to option values. - Setting extraSubstituters{ - this, {}, "extra-substituters", - R"( - Additional binary caches appended to those specified in - `substituters`. When used by unprivileged users, untrusted - substituters (i.e. those not listed in `trusted-substituters`) are - silently ignored. - )", - {"extra-binary-caches"}}; - Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9f21f0434..83d3a1fa1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1114,9 +1114,6 @@ std::list> getDefaultSubstituters() for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); - stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index eef01bde2..116dd6bfe 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -8,9 +8,18 @@ namespace nix { bool Config::set(const std::string & name, const std::string & value) { + bool append = false; auto i = _settings.find(name); - if (i == _settings.end()) return false; - i->second.setting->set(value); + if (i == _settings.end()) { + if (hasPrefix(name, "extra-")) { + i = _settings.find(std::string(name, 6)); + if (i == _settings.end() || !i->second.setting->isAppendable()) + return false; + append = true; + } else + return false; + } + i->second.setting->set(value, append); i->second.setting->overriden = true; return true; } @@ -180,6 +189,12 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } +template +bool BaseSetting::isAppendable() +{ + return false; +} + template void BaseSetting::convertToArg(Args & args, const std::string & category) { @@ -190,9 +205,18 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s); }}, }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s, true); }}, + }); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { value = str; } @@ -203,7 +227,7 @@ template<> std::string BaseSetting::to_string() const } template -void BaseSetting::set(const std::string & str) +void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); if (!string2Int(str, value)) @@ -217,7 +241,7 @@ std::string BaseSetting::to_string() const return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { if (str == "true" || str == "yes" || str == "1") value = true; @@ -248,9 +272,16 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { - value = tokenizeString(str); + auto ss = tokenizeString(str); + if (!append) value.clear(); + for (auto & s : ss) value.push_back(std::move(s)); +} + +template<> bool BaseSetting::isAppendable() +{ + return true; } template<> std::string BaseSetting::to_string() const @@ -258,7 +289,7 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { value = tokenizeString(str); } @@ -268,9 +299,9 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { - value.clear(); + if (!append) value.clear(); auto kvpairs = tokenizeString(str); for (auto & s : kvpairs) { @@ -281,6 +312,11 @@ template<> void BaseSetting::set(const std::string & str) } } +template<> bool BaseSetting::isAppendable() +{ + return true; +} + template<> std::string BaseSetting::to_string() const { Strings kvstrs; @@ -301,7 +337,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; -void PathSetting::set(const std::string & str) +void PathSetting::set(const std::string & str, bool append) { if (str == "") { if (allowEmpty) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1f5f4e7b9..71e31656d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -202,7 +202,10 @@ protected: assert(created == 123); } - virtual void set(const std::string & value) = 0; + virtual void set(const std::string & value, bool append = false) = 0; + + virtual bool isAppendable() + { return false; } virtual std::string to_string() const = 0; @@ -243,7 +246,9 @@ public: void operator =(const T & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; + + bool isAppendable() override; virtual void override(const T & v) { @@ -305,7 +310,7 @@ public: options->addSetting(this); } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; Path operator +(const char * p) const { return value + p; } From 7f56cf67bac3731ed8e217170eb548bf0fd2cfcb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Oct 2020 18:26:35 +0100 Subject: [PATCH 108/227] Fix assertion failure in tab completion for --option --- src/libmain/common-args.cc | 2 +- src/libutil/config.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9151a0344..3e4e475e5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, s.second.description); + completions->add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 116dd6bfe..be957dfe3 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -200,7 +200,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, - .description = description, + .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s); }}, @@ -209,7 +209,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) if (isAppendable()) args.addFlag({ .longName = "extra-" + name, - .description = description, + .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s, true); }}, @@ -260,13 +260,13 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, - .description = description, + .description = fmt("Enable the `%s` setting.", name), .category = category, .handler = {[=]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, - .description = description, + .description = fmt("Disable the `%s` setting.", name), .category = category, .handler = {[=]() { override(false); }} }); From b809c48ebba7b628a5fb0a0e284cf7068589d479 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 11:01:33 +0100 Subject: [PATCH 109/227] nix-shell.md: evaluated -> run Use "run" to avoid confusion with Nix evaluation. "evaluated" was intended to reference bash eval but it's ambiguous. --- doc/manual/src/command-ref/nix-shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index d1266930e..8c77923d0 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -32,7 +32,7 @@ URL of a tarball that will be downloaded and unpacked to a temporary location. The tarball must include a single top-level directory containing at least a file named `default.nix`. -If the derivation defines the variable `shellHook`, it will be evaluated +If the derivation defines the variable `shellHook`, it will be run after `$stdenv/setup` has been sourced. Since this hook is not executed by regular Nix builds, it allows you to perform initialisation specific to `nix-shell`. For example, the derivation attribute From d4c5d8d32a4bc5b3eaa63add23dca9d755dc5a74 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 11:12:28 +0100 Subject: [PATCH 110/227] nix-shell.md: Extend shellHook example --- doc/manual/src/command-ref/nix-shell.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 8c77923d0..88b675e71 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -41,10 +41,12 @@ to `nix-shell`. For example, the derivation attribute shellHook = '' echo "Hello shell" + export SOME_API_TOKEN="$(cat ~/.config/some-app/api-token)" ''; ``` -will cause `nix-shell` to print `Hello shell`. +will cause `nix-shell` to print `Hello shell` and set the `SOME_API_TOKEN` +environment variable to a user-configured value. # Options From dc5696b84f55a6706cddc3d747ef1aeffb564f43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Oct 2020 12:00:43 +0100 Subject: [PATCH 111/227] Fix test --- src/libutil/tests/config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c7777a21f..c305af9f5 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -80,8 +80,8 @@ namespace nix { class TestSetting : public AbstractSetting { public: TestSetting() : AbstractSetting("test", "test", {}) {} - void set(const std::string & value) {} - std::string to_string() const { return {}; } + void set(const std::string & value, bool append) override {} + std::string to_string() const override { return {}; } }; Config config; From c4d903ddb009aa6472530699e154d85a24eac51d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 20:55:53 +0100 Subject: [PATCH 112/227] Fix memory corruption caused by GC-invisible coroutine stacks Crucially this introduces BoehmGCStackAllocator, but it also adds a bunch of wiring to avoid making libutil depend on bdw-gc. Part of the solutions for #4178, #4200 --- src/libexpr/eval.cc | 26 ++++++++++++++++++++++++++ src/libexpr/local.mk | 2 +- src/libutil/serialise.cc | 35 ++++++++++++++++++++++++++++++++++- src/libutil/serialise.hh | 14 ++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4de87d647..ea7ba0a6a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -27,6 +27,10 @@ #include #include +#include +#include +#include + #endif namespace nix { @@ -220,6 +224,26 @@ static void * oomHandler(size_t requested) /* Convert this to a proper C++ exception. */ throw std::bad_alloc(); } + +class BoehmGCStackAllocator : public StackAllocator { + boost::coroutines2::protected_fixedsize_stack stack; + + public: + boost::context::stack_context allocate() override { + auto sctx = stack.allocate(); + GC_add_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + return sctx; + } + + void deallocate(boost::context::stack_context sctx) override { + GC_remove_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + stack.deallocate(sctx); + } + +}; + +static BoehmGCStackAllocator boehmGCStackAllocator; + #endif @@ -257,6 +281,8 @@ void initGC() GC_set_oom_fn(oomHandler); + StackAllocator::defaultAllocator = &boehmGCStackAllocator; + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 687a8ccda..a5422169d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS = +libexpr_LDFLAGS = -lboost_context ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5c9f6f901..28f6968d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,6 +171,39 @@ size_t StringSource::read(unsigned char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif +/* A concrete datatype allow virtual dispatch of stack allocation methods. */ +struct VirtualStackAllocator { + StackAllocator *allocator = StackAllocator::defaultAllocator; + + boost::context::stack_context allocate() { + return allocator->allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + allocator->deallocate(sctx); + } +}; + + +/* This class reifies the default boost coroutine stack allocation strategy with + a virtual interface. */ +class DefaultStackAllocator : public StackAllocator { + boost::coroutines2::default_stack stack; + + boost::context::stack_context allocate() { + return stack.allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + deallocate(sctx); + } +}; + +static DefaultStackAllocator defaultAllocatorSingleton; + +StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; + + std::unique_ptr sinkToSource( std::function fun, std::function eof) @@ -195,7 +228,7 @@ std::unique_ptr sinkToSource( size_t read(unsigned char * data, size_t len) override { if (!coro) - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](const unsigned char * data, size_t len) { if (len) yield(std::string((const char *) data, len)); }); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d7fe0b81e..5c7d3ce76 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +namespace boost::context { struct stack_context; } namespace nix { @@ -497,5 +498,18 @@ struct FramedSink : nix::BufferedSink }; }; +/* Stack allocation strategy for sinkToSource. + Mutable to avoid a boehm gc dependency in libutil. + + boost::context doesn't provide a virtual class, so we define our own. + */ +struct StackAllocator { + virtual boost::context::stack_context allocate() = 0; + virtual void deallocate(boost::context::stack_context sctx) = 0; + + /* The stack allocator to use in sinkToSource and potentially elsewhere. + It is reassigned by the initGC() method in libexpr. */ + static StackAllocator *defaultAllocator; +}; } From 2192cac634dcd13c3365f4dfeb5f393f4fd9e327 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 21:47:34 +0100 Subject: [PATCH 113/227] Fix RemoteStore pool deadlock in filterSource etc --- src/libstore/remote-store.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..ccbd3bb1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -471,9 +471,14 @@ ref RemoteStore::addCAToStore( worker_proto::write(*this, conn->to, references); conn->to << repair; - conn.withFramedSink([&](Sink & sink) { - dump.drainInto(sink); - }); + // The dump source may invoke the store, so we need to make some room. + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + } auto path = parseStorePath(readString(conn->from)); return readValidPathInfo(conn, path); From b43c13a9161daf1801188e61104debafa5243fe1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 23:18:24 +0100 Subject: [PATCH 114/227] BoehmGCStackAllocator: increase stack size to 8MB The default stack size was not based on the normal stack size and was too small. --- src/libexpr/eval.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ea7ba0a6a..486a9fc1a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -226,7 +226,12 @@ static void * oomHandler(size_t requested) } class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack; + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) + }; public: boost::context::stack_context allocate() override { From e8a45d07bccc315de88b8434393082eee6e944a8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 31 Oct 2020 23:52:09 +0100 Subject: [PATCH 115/227] Restore RestrictedStore.addToStoreFromDump implementation It was accidentally removed in commit ca30abb3fb36440e5a13161c39647189036fc18f --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3dacb218c..19d96dd8f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2074,6 +2074,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf return path; } + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + void narFromPath(const StorePath & path, Sink & sink) override { if (!goal.isAllowed(path)) From db5424bf09886afc1c81db36766522f68fc66ba8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 13:57:58 +0100 Subject: [PATCH 116/227] Don't send eval-related settings to the daemon --- src/libstore/remote-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..fb52ca6d0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -11,6 +11,7 @@ #include "finally.hh" #include "logging.hh" #include "callback.hh" +#include "filetransfer.hh" namespace nix { @@ -171,7 +172,8 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map overrides; - globalConfig.getSettings(overrides, true); + settings.getSettings(overrides, true); // libstore settings + fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); overrides.erase(settings.keepGoing.name); overrides.erase(settings.tryFallback.name); From 8b15650e7421cf9433f04c32ac73601aada1e3ab Mon Sep 17 00:00:00 2001 From: mkenigs <40775676+mkenigs@users.noreply.github.com> Date: Mon, 2 Nov 2020 09:32:05 -0600 Subject: [PATCH 117/227] docs: consistent console prompt (#4213) Everywhere else a $ is used --- doc/manual/src/installation/installing-binary.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 8b8d1d738..ae7fd458b 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -195,7 +195,7 @@ If you are comfortable navigating these tradeoffs, you can encrypt the volume with something along the lines of: ```console -alice$ diskutil apfs enableFileVault /nix -user disk +$ diskutil apfs enableFileVault /nix -user disk ``` ## Symlink the Nix store to a custom location @@ -234,13 +234,13 @@ as a helpful reference if you run into trouble. without a reboot: ```console - alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B ``` 3. Create the new APFS volume with diskutil: ```console - alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix + $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix ``` 4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't @@ -280,10 +280,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install` inside the binary tarball: ```console -alice$ cd /tmp -alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -alice$ cd nix-1.8-x86_64-darwin -alice$ ./install +$ cd /tmp +$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +$ cd nix-1.8-x86_64-darwin +$ ./install ``` If you need to edit the multi-user installation script to use different From 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 18:46:44 +0100 Subject: [PATCH 118/227] Don't use readDerivation() in addValidPath() readDerivation() requires a valid path. Fixes #4210. --- src/libstore/local-store.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bfad8fb21..2892b0407 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -623,7 +623,10 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(info.path); + auto drv = parseDerivation( + *this, + readFile(Store::toRealPath(info.path)), + Derivation::nameFromPath(info.path)); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1000,7 +1003,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, readDerivation(i.path)); + checkDerivationOutputs(i.path, + parseDerivation( + *this, + readFile(Store::toRealPath(i.path)), + Derivation::nameFromPath(i.path))); } /* Do a topological sort of the paths. This will throw an From 550e11f077ae508abde5a33998a9d4029880e7b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 19:07:37 +0100 Subject: [PATCH 119/227] nix repl: Fix handling of multi-line expressions --- src/nix/repl.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9ff386b1d..71794a309 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -212,7 +212,7 @@ void NixRepl::mainLoop(const std::vector & files) try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { - if (e.msg().find("unexpected $end") != std::string::npos) { + if (e.msg().find("unexpected end of file") != std::string::npos) { // For parse errors on incomplete input, we continue waiting for the next line of // input without clearing the input so far. continue; @@ -220,9 +220,9 @@ void NixRepl::mainLoop(const std::vector & files) printMsg(lvlError, e.msg()); } } catch (Error & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Interrupted & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } // We handled the current input fully, so we should clear it From 8cd2ff69c38913d5fcddeb0a012ba0cf34de0686 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 2 Nov 2020 15:50:14 -0500 Subject: [PATCH 120/227] nix-copy-closure: verify it works with drvs Creates test coverage for #4210 and 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 --- tests/nix-copy-closure.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index e5f6a0f12..1b63a3fca 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -7,14 +7,14 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { name = "nix-copy-closure"; nodes = { client = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgA ]; + virtualisation.pathsInNixDB = [ pkgA pkgD.drvPath ]; nix.binaryCaches = lib.mkForce [ ]; }; @@ -60,6 +60,12 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { # Copy the closure of package C via the SSH substituter. client.fail("nix-store -r ${pkgC}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix-copy-closure --to server --gzip ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + # FIXME # client.succeed( # "nix-store --option use-ssh-substituter true" From 797a52e31d97381e0be7e0a80e43f8cb259f8a6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:26:53 +0100 Subject: [PATCH 121/227] Add FIXME --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f1e2ab7f9..5e07261a6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,6 +90,8 @@ private: std::unique_ptr publicKeys; }; + // FIXME: get rid of recursive_mutex, it hides recursive SQLite + // queries. Sync _state; public: From e8c379555fa0441c5ab83b8e5a3a106d69fe2507 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:52:57 +0100 Subject: [PATCH 122/227] LocalStore: Get rid of recursive_mutex --- src/libstore/build/derivation-goal.cc | 6 +++--- src/libstore/local-store.cc | 23 ++++++++++++++++------- src/libstore/local-store.hh | 4 +--- src/libstore/path-info.hh | 2 +- src/nix-store/nix-store.cc | 2 +- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 19d96dd8f..bf2bad62c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3245,7 +3245,7 @@ void DerivationGoal::registerOutputs() if (!oldInfo.ultimate) { oldInfo.ultimate = true; worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); + worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3275,7 +3275,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); + worker.store.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3350,7 +3350,7 @@ void DerivationGoal::registerOutputs() { ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); + infos2.insert_or_assign(newInfo.path, newInfo); } worker.store.registerValidPaths(infos2); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2892b0407..e4e404dca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -7,6 +7,7 @@ #include "nar-info.hh" #include "references.hh" #include "callback.hh" +#include "topo-sort.hh" #include #include @@ -962,9 +963,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst void LocalStore::registerValidPath(const ValidPathInfo & info) { - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); + registerValidPaths({{info.path, info}}); } @@ -982,7 +981,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) SQLiteTxn txn(state->db); StorePathSet paths; - for (auto & i : infos) { + for (auto & [_, i] : infos) { assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); @@ -991,7 +990,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i.path); } - for (auto & i : infos) { + for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); @@ -1000,7 +999,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Check that the derivation outputs are correct. We can't do this in addValidPath() above, because the references might not be valid yet. */ - for (auto & i : infos) + for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, @@ -1014,7 +1013,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(paths); + topoSort(paths, + {[&](const StorePath & path) { + auto i = infos.find(path); + return i == infos.end() ? StorePathSet() : i->second.references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); txn.commit(); }); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5e07261a6..d4435220d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,9 +90,7 @@ private: std::unique_ptr publicKeys; }; - // FIXME: get rid of recursive_mutex, it hides recursive SQLite - // queries. - Sync _state; + Sync _state; public: diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 8ff5c466e..de87f8b33 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -107,6 +107,6 @@ struct ValidPathInfo virtual ~ValidPathInfo() { } }; -typedef list ValidPathInfos; +typedef std::map ValidPathInfos; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 14baabc36..6c2702bfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -516,7 +516,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) info->narHash = hash.first; info->narSize = hash.second; } - infos.push_back(std::move(*info)); + infos.insert_or_assign(info->path, *info); } } From 5e6eabe1551f3fa2a463fced39a4a48b504ed1ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 14:44:52 +0100 Subject: [PATCH 123/227] Fix error message 'assertion failed at' --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4de87d647..cb8e0a0de 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1405,7 +1405,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); + throwAssertionError(pos, "assertion '%1%' failed", out.str()); } body->eval(state, env, v); } From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 21 Oct 2020 21:31:19 +0200 Subject: [PATCH 124/227] Allow substituting paths when building remotely using `ssh-ng://` Until now, it was not possible to substitute missing paths from e.g. `https://cache.nixos.org` on a remote server when building on it using the new `ssh-ng` protocol. This is because every store implementation except legacy `ssh://` ignores the substitution flag passed to `Store::queryValidPaths` while the `legacy-ssh-store` substitutes the remote store using `cmdQueryValidPaths` when the remote store is opened with `nix-store --serve`. This patch slightly modifies the daemon protocol to allow passing an integer value suggesting whether to substitute missing paths during `wopQueryValidPaths`. To implement this on the daemon-side, the substitution logic from `nix-store --serve` has been moved into a protected method named `Store::substitutePaths` which gets currently called from `LocalStore::queryValidPaths` and `Store::queryValidPaths` if `maybeSubstitute` is `true`. Fixes #2770 --- flake.nix | 3 ++- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/remote-store.cc | 3 +++ src/libstore/store-api.cc | 22 ++++++++++++++++++++++ src/libstore/store-api.hh | 5 +++++ src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 23 +---------------------- 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/flake.nix b/flake.nix index 2abbdff53..8ff048be3 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ versionSuffix = if officialRelease then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}"; + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; officialRelease = false; @@ -117,6 +117,7 @@ nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { name = "nix-${version}"; + inherit version; src = self; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4dbc7ba38..60cca4fda 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryValidPaths: { auto paths = worker_proto::read(*store, from, Phantom {}); + + SubstituteFlag substitute = NoSubstitute; + if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { + substitute = readInt(from) ? Substitute : NoSubstitute; + } + logger->startWork(); - auto res = store->queryValidPaths(paths); + if (substitute) { + store->substitutePaths(paths); + } + auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); worker_proto::write(*store, to, res); break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..c27a0f278 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute } else { conn->to << wopQueryValidPaths; worker_proto::write(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { + conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + } conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..3129d6637 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath, } +void Store::substitutePaths(const StorePathSet & paths) +{ + std::vector paths2; + for (auto & path : paths) + if (!path.isDerivation()) + paths2.push_back({path}); + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + queryMissing(paths2, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willSubstitute.empty()) + try { + std::vector subs; + for (auto & p : willSubstitute) subs.push_back({p}); + buildPaths(subs); + } catch (Error & e) { + logWarning(e.info()); + } +} + + StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..9a373b561 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,6 +360,11 @@ protected: public: + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + void substitutePaths(const StorePathSet & paths); + /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b3705578e..63bd6ea49 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11a +#define PROTOCOL_VERSION 0x11b #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6c2702bfe..54394e921 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) store->addTempRoot(path); - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ if (substitute && writeAllowed) { - /* Filter out .drv files (we don't want to build anything). */ - std::vector paths2; - for (auto & path : paths) - if (!path.isDerivation()) - paths2.push_back({path}); - uint64_t downloadSize, narSize; - StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths2, - willBuild, willSubstitute, unknown, downloadSize, narSize); - /* FIXME: should use ensurePath(), but it only - does one path at a time. */ - if (!willSubstitute.empty()) - try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); - store->buildPaths(subs); - } catch (Error & e) { - logWarning(e.info()); - } + store->substitutePaths(paths); } worker_proto::write(*store, out, store->queryValidPaths(paths)); From b87f84cf55b4ca666b31c511e2489789e3322da4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 15:04:34 +0100 Subject: [PATCH 125/227] Fix appending to Setting Fixes: warning: unknown setting 'extra-sandbox-paths' --- src/libutil/config.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index be957dfe3..7af3e7883 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -291,7 +291,14 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str, bool append) { - value = tokenizeString(str); + if (!append) value.clear(); + for (auto & s : tokenizeString(str)) + value.insert(s); +} + +template<> bool BaseSetting::isAppendable() +{ + return true; } template<> std::string BaseSetting::to_string() const @@ -302,9 +309,7 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str, bool append) { if (!append) value.clear(); - auto kvpairs = tokenizeString(str); - for (auto & s : kvpairs) - { + for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); From fb7735e4cf3a1ee6337bf1f2ee15204bb11304b2 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Sat, 7 Nov 2020 15:00:22 +0100 Subject: [PATCH 126/227] nix develop: Preserve stdin with `-c` --- src/nix/develop.cc | 4 ++-- tests/nix-shell.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8fea7ee9c..457d94382 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -474,9 +474,9 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } - // If running a phase, don't want an interactive shell running after + // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile - auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath} + auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 1228bb04f..dfe8ed0c1 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,6 +58,7 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' +echo foo | nix develop -f shell.nix shellDrv -c cat # preserve stdin with `-c` # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) From 579b953231599f111dcc9fac95491458c9f6ef08 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Mon, 9 Nov 2020 17:50:51 +0100 Subject: [PATCH 127/227] Make test case more precise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt --- tests/nix-shell.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index dfe8ed0c1..3b000a049 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,7 +58,8 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' -echo foo | nix develop -f shell.nix shellDrv -c cat # preserve stdin with `-c` +# Preserve stdin with `-c` +echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) From 3f24a417dacd5a5964f781f4d704a5253401d380 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Mon, 9 Nov 2020 17:57:39 +0100 Subject: [PATCH 128/227] Add test case for incidentally fixed #4228 --- tests/nix-shell.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 3b000a049..7b2be650a 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,9 +58,13 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' -# Preserve stdin with `-c` + +# Ensure `nix develop -c` preserves stdin echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo +# Ensure `nix develop -c` actually executes the command if stdout isn't a terminal +nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo + # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) [[ -n $stdenv ]] From 0ed7c957bed18d963df65d55f0c0ffc79dee656d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 23:21:55 +0000 Subject: [PATCH 129/227] Bump cachix/install-nix-action from v11 to v12 (#4237) Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v11 to v12. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v11...07da2520eebede906fbeefa9dd0a2b635323909d) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 829111b67..021642f4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v11 + - uses: cachix/install-nix-action@v12 #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) From 108a2dab7e460533064b24f5dff05adc453acb27 Mon Sep 17 00:00:00 2001 From: "Ricardo M. Correia" Date: Tue, 10 Nov 2020 04:24:55 +0100 Subject: [PATCH 130/227] Fix stack overflow introduced in #4206 --- src/libutil/serialise.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 28f6968d0..038ede049 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -195,7 +195,7 @@ class DefaultStackAllocator : public StackAllocator { } void deallocate(boost::context::stack_context sctx) { - deallocate(sctx); + stack.deallocate(sctx); } }; From 4864df6d6b7f5fc5c0815bf7733c40663ea856d7 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 10 Nov 2020 08:48:49 -0600 Subject: [PATCH 131/227] enable Darwin.arm64 to install x86_64 binary Throwing @thefloweringash under the bus if this doesn't work, but it sounds like Apple Silicon devices can use the x86_64 binary for now. Fixes #4058 --- scripts/install.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install.in b/scripts/install.in index 39fae37e3..9a281d776 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -29,6 +29,8 @@ case "$(uname -s).$(uname -m)" in Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;; + Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From 4badb6943f73e3d740d4de793eb9860e26842bc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Nov 2020 23:22:45 +0100 Subject: [PATCH 132/227] Fix use of dirty Git/Mercurial inputs with chroot stores Fixes: $ nix build --store /tmp/nix /home/eelco/Dev/patchelf#hydraJobs.build.x86_64-linux warning: Git tree '/home/eelco/Dev/patchelf' is dirty error: --- RestrictedPathError ------------------------------------------------------------------------------------------- nix access to path '/tmp/nix/nix/store/xmkvfmffk7xfnazykb5kx999aika8an4-source/flake.nix' is forbidden in restricted mode (use '--show-trace' to show detailed location information) --- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a6411b02b..e7712c5fd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -273,7 +273,7 @@ struct GitInputScheme : InputScheme haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return { - Tree(store->printStorePath(storePath), std::move(storePath)), + Tree(store->toRealPath(storePath), std::move(storePath)), input }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 7d3d52751..41f60c45c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -166,7 +166,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return { - Tree(store->printStorePath(storePath), std::move(storePath)), + Tree(store->toRealPath(storePath), std::move(storePath)), input }; } From 8abb80a478116b10bf37162c71f602262de412a9 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Thu, 22 Oct 2020 23:59:01 -0500 Subject: [PATCH 133/227] Print built derivations as json for build Add --json option to nix build to allow machine readable output on stdout with all built derivations Fixes #1930 --- src/nix/build.cc | 6 +++++- src/nix/installables.cc | 27 +++++++++++++++++++++++++++ src/nix/installables.hh | 5 +++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 65708e98b..67be4024b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,9 +5,11 @@ #include "store-api.hh" #include "local-fs-store.hh" +#include + using namespace nix; -struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile +struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; BuildMode buildMode = bmNormal; @@ -86,6 +88,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile }, buildables[i]); updateProfile(buildables); + + if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 7473c9758..f385289e5 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -16,8 +16,35 @@ #include #include +#include + namespace nix { +nlohmann::json BuildableOpaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuildableFromDrv::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = path ? store->printStorePath(*path) : ""; + } + return res; +} + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const Buildable & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable); + } + return res; +} + void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, diff --git a/src/nix/installables.hh b/src/nix/installables.hh index c7c2f8981..f37b3f829 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -7,6 +7,8 @@ #include +#include + namespace nix { struct DrvInfo; @@ -16,11 +18,13 @@ namespace eval_cache { class EvalCache; class AttrCursor; } struct BuildableOpaque { StorePath path; + nlohmann::json toJSON(ref store) const; }; struct BuildableFromDrv { StorePath drvPath; std::map> outputs; + nlohmann::json toJSON(ref store) const; }; typedef std::variant< @@ -29,6 +33,7 @@ typedef std::variant< > Buildable; typedef std::vector Buildables; +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); struct App { From d52b12c0a53f88b8ea9238d604f293b54c8ae51a Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Fri, 23 Oct 2020 08:20:38 -0500 Subject: [PATCH 134/227] Test nix build --json --- tests/build.sh | 12 ++++++++++++ tests/local.mk | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/build.sh diff --git a/tests/build.sh b/tests/build.sh new file mode 100644 index 000000000..aa54b88eb --- /dev/null +++ b/tests/build.sh @@ -0,0 +1,12 @@ +source common.sh + +expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' +nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs.first | match(".*multiple-outputs-a-first")) and + (.outputs.second | match(".*multiple-outputs-a-second"))) + and (.[1] | + (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs.out | match(".*multiple-outputs-b"))) +' diff --git a/tests/local.mk b/tests/local.mk index a1929f96d..ce94ec80e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -35,7 +35,8 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ - content-addressed.sh + content-addressed.sh \ + build.sh # parallel.sh # build-remote-content-addressed-fixed.sh \ From c4c3c15c19bc448a4797e5d9577539cc14890618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20H=C3=B6ppner?= Date: Thu, 12 Nov 2020 15:46:08 +0000 Subject: [PATCH 135/227] Fix default nix-path The default nix-path values for nixpkgs and root channels were incorrect. --- src/libexpr/eval.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5e3fcf4ac..c6f4d1716 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2104,10 +2104,19 @@ EvalSettings::EvalSettings() Strings EvalSettings::getDefaultNixPath() { Strings res; - auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathExists(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + add(getHome() + "/.nix-defexpr/channels"); - add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); - add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); + add(settings.nixStateDir + "/profiles/per-user/root/channels"); return res; } From ac5081d28035ef8d6a907fc3bff812d534719f74 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Nov 2020 17:49:27 +0100 Subject: [PATCH 136/227] nix-build: Fix #4197 output order regression --- src/nix-build/nix-build.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f60e0706c..74fafd426 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -487,6 +487,7 @@ static void main_nix_build(int argc, char * * argv) else { std::vector pathsToBuild; + std::vector> pathsToBuildOrdered; std::map> drvMap; @@ -498,6 +499,7 @@ static void main_nix_build(int argc, char * * argv) throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); pathsToBuild.push_back({drvPath, {outputName}}); + pathsToBuildOrdered.push_back({drvPath, {outputName}}); auto i = drvMap.find(drvPath); if (i != drvMap.end()) @@ -513,25 +515,23 @@ static void main_nix_build(int argc, char * * argv) std::vector outPaths; - for (auto & [drvPath, info] : drvMap) { - auto & [counter, wantedOutputs] = info; + for (auto & [drvPath, outputName] : pathsToBuildOrdered) { + auto & [counter, _wantedOutputs] = drvMap.at({drvPath}); std::string drvPrefix = outLink; if (counter) drvPrefix += fmt("-%d", counter + 1); auto builtOutputs = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : wantedOutputs) { - auto outputPath = builtOutputs.at(outputName); + auto outputPath = builtOutputs.at(outputName); - if (auto store2 = store.dynamic_pointer_cast()) { - std::string symlink = drvPrefix; - if (outputName != "out") symlink += "-" + outputName; - store2->addPermRoot(outputPath, absPath(symlink)); - } - - outPaths.push_back(outputPath); + if (auto store2 = store.dynamic_pointer_cast()) { + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; + store2->addPermRoot(outputPath, absPath(symlink)); } + + outPaths.push_back(outputPath); } logger->stop(); From d264da8d968bb3afe29def1f4aff0f91f25e36c6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Nov 2020 17:49:55 +0100 Subject: [PATCH 137/227] tests: Test #4197 nix-build output order regression --- tests/nix-build-examples.nix | 33 +++++++++++++++++++++++++++++++++ tests/nix-build.sh | 15 +++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tests/nix-build-examples.nix diff --git a/tests/nix-build-examples.nix b/tests/nix-build-examples.nix new file mode 100644 index 000000000..e54dbbf62 --- /dev/null +++ b/tests/nix-build-examples.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +rec { + + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + + input1 = mkDerivation { + name = "dependencies-input-1"; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + ''; + }; + + body = mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + meta.description = "Random test package"; + }; + +} diff --git a/tests/nix-build.sh b/tests/nix-build.sh index 3123c6da3..44a5a14cd 100644 --- a/tests/nix-build.sh +++ b/tests/nix-build.sh @@ -26,3 +26,18 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out) [[ $outPath = $outPath2 ]] + +# The order of the paths on stdout must correspond to the -A options +# https://github.com/NixOS/nix/issues/4197 + +input0="$(nix-build nix-build-examples.nix -A input0 --no-out-link)" +input1="$(nix-build nix-build-examples.nix -A input1 --no-out-link)" +input2="$(nix-build nix-build-examples.nix -A input2 --no-out-link)" +body="$(nix-build nix-build-examples.nix -A body --no-out-link)" + +outPathsA="$(echo $(nix-build nix-build-examples.nix -A input0 -A input1 -A input2 -A body --no-out-link))" +[[ "$outPathsA" = "$input0 $input1 $input2 $body" ]] + +# test a different ordering to make sure it fails, not just in 23 out of 24 permutations +outPathsB="$(echo $(nix-build nix-build-examples.nix -A body -A input1 -A input2 -A input0 --no-out-link))" +[[ "$outPathsB" = "$body $input1 $input2 $input0" ]] From 01db455733899728947a15f8d50942f10432d61e Mon Sep 17 00:00:00 2001 From: Jake Waksbaum Date: Mon, 16 Nov 2020 02:35:50 -0500 Subject: [PATCH 138/227] Fix deadlock in nix-store when max-connections=1 This fixes a bug I encountered where `nix-store -qR` will deadlock when the `--include-outputs` flag is passed and `max-connections=1`. The deadlock occurs because `RemoteStore::queryDerivationOutputs` takes the only connection from the connection pool and uses it to check the daemon version. If the version is new enough, it calls `Store::queryDerivationOutputs`, which eventually calls `RemoteStore::queryPartialDerivationOutputMap`, where we take another connection from the connection pool to check the version again. Because we still haven't released the connection from the caller, this waits for a connection to be available, causing a deadlock. This diff solves the issue by using `getProtocol` to check the protocol version in the caller `RemoteStore::queryDerivationOutputs`, which immediately frees the connection back to the pool before returning the protocol version. That way we've already freed the connection by the time we call `RemoteStore::queryPartialDerivationOutputMap`. --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..a392f3b8c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -409,10 +409,10 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { return Store::queryDerivationOutputs(path); } + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); From ef84c780bb901011e090b9f12d293d136193a428 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Nov 2020 16:26:29 +0100 Subject: [PATCH 139/227] filterANSIEscapes(): Handle UTF-8 characters --- src/libutil/tests/tests.cc | 10 ++++++++++ src/libutil/util.cc | 23 ++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index ffba832d8..35a5d27bb 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -587,4 +587,14 @@ namespace nix { ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); } + + TEST(filterANSIEscapes, utf8) { + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); + } + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 53342b5cb..01ab9111f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1409,7 +1409,28 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in i++; else { - t += *i++; w++; + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; } } From 7de21f6664ffd4a66e5ef70058a4c450985a1981 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 13 Nov 2020 17:00:32 +0100 Subject: [PATCH 140/227] Make the sql debug statements more useful Print the expanded sql query (with the variables bound to their value) rather than the original one in case of error --- src/libstore/sqlite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 31a1f0cac..f5935ee5c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -147,14 +147,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } From bccff827dc968b08bddda03aadcb3d9cc41c2719 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:50:36 +0100 Subject: [PATCH 141/227] Fix deadlock in IFD through the daemon Fixes #4235. --- src/libstore/derivations.cc | 6 ++++-- tests/remote-store.sh | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 517ecfaa2..eea129df3 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -463,13 +463,15 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath { auto h = drvHashes.find(drvPath); if (h == drvHashes.end()) { - assert(store.isValidPath(drvPath)); // Cache it h = drvHashes.insert_or_assign( drvPath, hashDerivationModulo( store, - store.readDerivation(drvPath), + parseDerivation( + store, + readFile(store.toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)), false)).first; } return h->second; diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 3a61946f9..f7ae1a2ed 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -7,6 +7,20 @@ nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor startDaemon +# Test import-from-derivation through the daemon. +[[ $(nix eval --impure --raw --expr ' + with import ./config.nix; + import ( + mkDerivation { + name = "foo"; + bla = import ./dependencies.nix; + buildCommand = " + echo \\\"hi\\\" > $out + "; + } + ) +') = hi ]] + storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh nix-store --dump-db > $TEST_ROOT/d1 From e6b7c7b79c697f1d8508930964e8c810f04a8963 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:58:55 +0100 Subject: [PATCH 142/227] Cleanup --- src/libstore/derivations.cc | 5 +---- src/libstore/local-store.cc | 10 ++-------- src/libstore/store-api.cc | 8 ++++++++ src/libstore/store-api.hh | 3 +++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index eea129df3..106ec5daa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -468,10 +468,7 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath drvPath, hashDerivationModulo( store, - parseDerivation( - store, - readFile(store.toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)), + store.readInvalidDerivation(drvPath), false)).first; } return h->second; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4e404dca..93d073768 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -624,10 +624,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = parseDerivation( - *this, - readFile(Store::toRealPath(info.path)), - Derivation::nameFromPath(info.path)); + auto drv = readInvalidDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1003,10 +1000,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, - parseDerivation( - *this, - readFile(Store::toRealPath(i.path)), - Derivation::nameFromPath(i.path))); + readInvalidDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..7f808f45a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1007,6 +1007,14 @@ Derivation Store::readDerivation(const StorePath & drvPath) } } +Derivation Store::readInvalidDerivation(const StorePath & drvPath) +{ + return parseDerivation( + *this, + readFile(Store::toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)); +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..30c9e7d0a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -611,6 +611,9 @@ public: /* Read a derivation (which must already be valid). */ Derivation readDerivation(const StorePath & drvPath); + /* Read a derivation from a potentially invalid path. */ + Derivation readInvalidDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If From 3daa256728e29f9059a79b8ecaf9325e52f1704a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 15:26:39 +0100 Subject: [PATCH 143/227] Remove tests.binaryTarball This test no longer works on Hydra because import-from-derivation is no longer allowed. --- flake.nix | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/flake.nix b/flake.nix index 8ff048be3..653cacded 100644 --- a/flake.nix +++ b/flake.nix @@ -388,38 +388,6 @@ inherit (self) overlay; }); - # Test whether the binary tarball works in an Ubuntu system. - tests.binaryTarball = - with nixpkgsFor.x86_64-linux; - vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu1204x86_64; - } - '' - set -x - useradd -m alice - su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*' - mkdir /dest-nix - mount -o bind /dest-nix /nix # Provide a writable /nix. - chown alice /nix - su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' - su - alice -c 'nix-store --verify' - su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}' - - # Check whether 'nix upgrade-nix' works. - cat > /tmp/paths.nix < Date: Tue, 17 Nov 2020 15:36:20 +0100 Subject: [PATCH 144/227] Remove stray debug statement This was causing a failure on macOS. https://hydra.nixos.org/build/130354318 --- tests/binary-cache.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 6415c9302..e3b3982fe 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -258,9 +258,6 @@ docPath=$(nix-store -q --references $outPath) # +---...-multi-output-doc nix copy --to "file://$cacheDir" $outPath -( echo $outPath $docPath - find $cacheDir -) >/tmp/blurb hashpart() { basename "$1" | cut -c1-32 From 0fa6d380b2143af68d4c7c6ab55b639516014304 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 18 Nov 2020 11:20:50 +0700 Subject: [PATCH 145/227] fix typo in comment in fetchurl.nix --- corepkgs/fetchurl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index a84777f57..02531103b 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -1,6 +1,6 @@ { system ? "" # obsolete , url -, hash ? "" # an SRI ash +, hash ? "" # an SRI hash # Legacy hash specification , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" From 2113ae2d856a208350ccbafdc19e8dda322515b8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Nov 2020 16:50:06 +0000 Subject: [PATCH 146/227] Make drv hash modulo memo table thread-safe Let's get one step closer to the daemon not needing to fork. --- src/libexpr/primops.cc | 7 ++++--- src/libstore/derivations.cc | 27 +++++++++++++++------------ src/libstore/derivations.hh | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 236433ef1..41f06c219 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1132,9 +1132,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * However, we don't bother doing this for floating CA derivations because their "hash modulo" is indeterminate until built. */ - if (drv.type() != DerivationType::CAFloating) - drvHashes.insert_or_assign(drvPath, - hashDerivationModulo(*state.store, Derivation(drv), false)); + if (drv.type() != DerivationType::CAFloating) { + auto h = hashDerivationModulo(*state.store, Derivation(drv), false); + drvHashes.lock()->insert_or_assign(drvPath, h); + } state.mkAttrs(v, 1 + drv.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 106ec5daa..231ca26c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -451,7 +451,7 @@ DerivationType BasicDerivation::type() const } -DrvHashes drvHashes; +Sync drvHashes; /* pathDerivationModulo and hashDerivationModulo are mutually recursive */ @@ -459,19 +459,22 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - auto h = drvHashes.find(drvPath); - if (h == drvHashes.end()) { - // Cache it - h = drvHashes.insert_or_assign( - drvPath, - hashDerivationModulo( - store, - store.readInvalidDerivation(drvPath), - false)).first; + { + auto hashes = drvHashes.lock(); + auto h = hashes->find(drvPath); + if (h != hashes->end()) { + return h->second; + } } - return h->second; + auto h = hashDerivationModulo( + store, + store.readInvalidDerivation(drvPath), + false); + // Cache it + drvHashes.lock()->insert_or_assign(drvPath, h); + return h; } /* See the header for interface details. These are the implementation details. diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f9ba935e6..b966d6d90 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -210,7 +210,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; -extern DrvHashes drvHashes; // FIXME: global, not thread-safe +// FIXME: global, though at least thread-safe. +extern Sync drvHashes; /* Memoisation of `readDerivation(..).resove()`. */ typedef std::map< From 0327580e5485143acb6eb7c8c515e3e179f1e3fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Nov 2020 20:30:41 +0100 Subject: [PATCH 147/227] Fix assertion failure in LockFile::LockFile() Fixes #4241. --- src/libexpr/flake/lockfile.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 8e2f7131f..6089d1363 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -78,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) { if (jsonNode.find("inputs") == jsonNode.end()) return; for (auto & i : jsonNode["inputs"].items()) { - if (i.value().is_array()) { + if (i.value().is_array()) { // FIXME: remove, obsolete InputPath path; for (auto & j : i.value()) path.push_back(j); @@ -87,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) std::string inputKey = i.value(); auto k = nodeMap.find(inputKey); if (k == nodeMap.end()) { - auto jsonNode2 = json["nodes"][inputKey]; - auto input = std::make_shared(jsonNode2); + auto nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = std::make_shared(*jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; - getInputs(*input, jsonNode2); + getInputs(*input, *jsonNode2); } if (auto child = std::dynamic_pointer_cast(k->second)) node.inputs.insert_or_assign(i.key(), child); From 4dcb183af31d5cb33b6ef8e581e77d1c892a58b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Nov 2020 20:59:36 +0100 Subject: [PATCH 148/227] AttrCursor::getStringWithContext(): Force re-evaluation if the cached context is not valid Fixes #4236. --- src/libexpr/eval-cache.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 381344b40..7b025be23 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -525,8 +525,17 @@ string_t AttrCursor::getStringWithContext() cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { - debug("using cached string attribute '%s'", getAttrPathStr()); - return *s; + bool valid = true; + for (auto & c : s->second) { + if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) { + valid = false; + break; + } + } + if (valid) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } } else throw TypeError("'%s' is not a string", getAttrPathStr()); } From c3bad73e27a5bda3f8f8c768cec818c52b0f95e3 Mon Sep 17 00:00:00 2001 From: Wil Taylor Date: Sat, 21 Nov 2020 14:28:49 +1000 Subject: [PATCH 149/227] Added switch --- src/nix/bundle.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2d0a0b6ea..c59018726 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -11,6 +11,7 @@ struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; std::optional outLink; + bool skipReferenceCheck = false; CmdBundle() { @@ -32,6 +33,12 @@ struct CmdBundle : InstallableCommand .handler = {&outLink}, .completer = completePath }); + + addFlag({ + .longName = "skip-refcheck", + .description = "Skip checking of references in bundle.", + .handler = {&skipReferenceCheck, true}, + }); } std::string description() override @@ -117,7 +124,7 @@ struct CmdBundle : InstallableCommand auto outPathS = store->printStorePath(outPath); auto info = store->queryPathInfo(outPath); - if (!info->references.empty()) + if (!info->references.empty() && !skipReferenceCheck) throw Error("'%s' has references; a bundler must not leave any references", outPathS); if (!outLink) From 233b61d3d6482544c35b9d340240bf3260acff13 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 6 Nov 2020 11:53:00 +0100 Subject: [PATCH 150/227] installer: simplify the per-build installation The goal is to allow the installation and testing of arbitrary Nix versions. Extend the base installer to accept a `--tarball-url-prefix ` to change where the Nix tarball is getting downloaded from. Once this is merged it should allow to: 1. Pick an evaluation at https://hydra.nixos.org/jobset/nix/master that looks healthy 2. Select the installedScript build and find the store path. Now equipped with all of this, use an instance of nar-serve to fetch the install script and release tarballs: curl -sfL https://nar-serve.numtide.com/nix/store/rkv4yh7pym941bhj0849zqdkg2546bdv-installer-script/install \ | sh --tarball-url-prefix https://nar-serve.numtide.com/nix/store Or with cachix, strip the /nix/store and derivation name and then: curl -sfL https://mycache.cachix.org/serve/rkv4yh7pym941bhj0849zqdkg2546bdv/install \ | sh --tarball-url-prefix https://mycache.cachix.org/serve Fixes #4047 --- flake.nix | 19 +++++++++++- scripts/install-nix-from-closure.sh | 3 ++ scripts/install.in | 46 ++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 653cacded..7311a2471 100644 --- a/flake.nix +++ b/flake.nix @@ -324,9 +324,26 @@ '' mkdir -p $out/nix-support + # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix + # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ") + (system: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + '' + ) [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${version} diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 2de41a6a4..6352a8fac 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -98,6 +98,9 @@ while [ $# -gt 0 ]; do echo "" echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" echo "" + if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then + echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from." + fi ) >&2 # darwin and Catalina+ diff --git a/scripts/install.in b/scripts/install.in index 9a281d776..7c3b795cc 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -25,18 +25,47 @@ require_util() { } case "$(uname -s).$(uname -m)" in - Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; - Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; - Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; - Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; - # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;; - Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + Linux.x86_64) + hash=@tarballHash_x86_64-linux@ + path=@tarballPath_x86_64-linux@ + system=x86_64-linux + ;; + Linux.i?86) + hash=@tarballHash_i686-linux@ + path=@tarballPath_i686-linux@ + system=i686-linux + ;; + Linux.aarch64) + hash=@tarballHash_aarch64-linux@ + path=@tarballPath_aarch64-linux@ + system=aarch64-linux + ;; + Darwin.x86_64) + hash=@tarballHash_x86_64-darwin@ + path=@tarballPath_x86_64-darwin@ + system=x86_64-darwin + ;; + Darwin.arm64) + hash=@binaryTarball_x86_64-darwin@ + path=@tarballPath_x86_64-darwin@ + # eventually maybe: arm64-darwin + system=x86_64-darwin + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" +# Use this command-line option to fetch the tarballs using nar-serve or Cachix +if "${1:---tarball-url-prefix}"; then + if [ -z "${2:-}" ]; then + oops "missing argument for --tarball-url-prefix" + fi + url=${2}/${path} + shift 2 +else + url=https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz +fi -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" +tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" @@ -68,6 +97,7 @@ tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) [ -e "$script" ] || oops "installation script is missing from the binary tarball!" +export INVOKED_FROM_INSTALL_IN=1 "$script" "$@" } # End of wrapping From df83b6df68a7a7d71747c1d6cfb98f6b4ee732f0 Mon Sep 17 00:00:00 2001 From: Kai Wohlfahrt Date: Sat, 21 Nov 2020 22:06:15 +0000 Subject: [PATCH 151/227] Return signatures in Perl path info --- perl/lib/Nix/Store.xs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..9e3b7d389 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -110,10 +110,14 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); - AV * arr = newAV(); + AV * refs = newAV(); for (auto & i : info->references) - av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) arr))); + av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); + XPUSHs(sv_2mortal(newRV((SV *) refs))); + AV * sigs = newAV(); + for (auto & i : info->sigs) + av_push(sigs, newSVpv(i.c_str(), 0)); + XPUSHs(sv_2mortal(newRV((SV *) sigs))); } catch (Error & e) { croak("%s", e.what()); } From 07603890d2867907905ba411cee550390d868936 Mon Sep 17 00:00:00 2001 From: Wil Taylor Date: Mon, 23 Nov 2020 21:19:40 +1000 Subject: [PATCH 152/227] Removed reference check from bundler command --- src/nix/bundle.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index c59018726..eddd82f40 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -11,7 +11,6 @@ struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; std::optional outLink; - bool skipReferenceCheck = false; CmdBundle() { @@ -34,11 +33,6 @@ struct CmdBundle : InstallableCommand .completer = completePath }); - addFlag({ - .longName = "skip-refcheck", - .description = "Skip checking of references in bundle.", - .handler = {&skipReferenceCheck, true}, - }); } std::string description() override @@ -123,10 +117,6 @@ struct CmdBundle : InstallableCommand auto outPathS = store->printStorePath(outPath); - auto info = store->queryPathInfo(outPath); - if (!info->references.empty() && !skipReferenceCheck) - throw Error("'%s' has references; a bundler must not leave any references", outPathS); - if (!outLink) outLink = baseNameOf(app.program); From 226116f48272dbe25bb9d3b8fa138cf6b10ec8ef Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 23 Nov 2020 16:12:33 +0000 Subject: [PATCH 153/227] fetchMercurial: set HGPLAIN when invoking hg Without setting HGPLAIN, the user's environment leaks into hg invocations, which means that the output may not be in the expected format. HGPLAIN is the Mercurial-recommended solution for this in that it's intended for uses by scripts and programs which are looking to parse Mercurial's output in a consistent manner. --- src/libfetchers/mercurial.cc | 54 ++++++++++++++++++++++++++++-------- tests/fetchMercurial.sh | 3 ++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 41f60c45c..07a51059d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -11,6 +11,36 @@ using namespace std::string_literals; namespace nix::fetchers { +namespace { + +RunOptions hgOptions(const Strings & args) { + RunOptions opts("hg", args); + opts.searchPath = true; + + auto env = getEnv(); + // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc. + env["HGPLAIN"] = ""; + opts.environment = env; + + return opts; +} + +// runProgram wrapper that uses hgOptions instead of stock RunOptions. +string runHg(const Strings & args, const std::optional & input = {}) +{ + RunOptions opts = hgOptions(args); + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + + return res.second; +} + +} + struct MercurialInputScheme : InputScheme { std::optional inputFromURL(const ParsedURL & url) override @@ -100,11 +130,11 @@ struct MercurialInputScheme : InputScheme assert(sourcePath); // FIXME: shut up if file is already tracked. - runProgram("hg", true, + runHg( { "add", *sourcePath + "/" + std::string(file) }); if (commitMsg) - runProgram("hg", true, + runHg( { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } @@ -130,7 +160,7 @@ struct MercurialInputScheme : InputScheme if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { - bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; if (!clean) { @@ -143,10 +173,10 @@ struct MercurialInputScheme : InputScheme if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); + input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, actualUrl)); @@ -224,33 +254,33 @@ struct MercurialInputScheme : InputScheme if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) + hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); if (pathExists(cacheDir)) { try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + runHg({ "recover", "-R", cacheDir }); + runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } else { throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir }); } } auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); @@ -263,7 +293,7 @@ struct MercurialInputScheme : InputScheme Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); + runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index af8ef8d5b..d8a4e09d2 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -15,6 +15,9 @@ hg init $repo echo '[ui]' >> $repo/.hg/hgrc echo 'username = Foobar ' >> $repo/.hg/hgrc +# Set ui.tweakdefaults to ensure HGPLAIN is being set. +echo 'tweakdefaults = True' >> $repo/.hg/hgrc + echo utrecht > $repo/hello touch $repo/.hgignore hg add --cwd $repo hello .hgignore From 5b0790355fe10b5cdc2468928a7cb4703cd0861a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 23 Nov 2020 17:40:17 +0100 Subject: [PATCH 154/227] Fix macOS sandbox build Since c4c3c15c19bc448a4797e5d9577539cc14890618 (#4251) building Nix for macOS with sandboxing fails: ``` getting status of /nix/var/nix/profiles/per-user/root/channels/nixpkgs: Operation not permitted ``` This happens, because `EvalSettings::getDefaultNixPath` tries to access paths outside the sandbox. Since the state-dir is not required for doc generation, it is set to the dummy folder. This needs to be done for all nix invocations during doc generation, as `EvalSettings::getDefaultNixPath` is called unconditionally. --- doc/manual/local.mk | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 7d9a1a3e8..bb8b3b60a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -15,7 +15,14 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 dist-files += $(man-pages) -nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr +# Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. +dummy-env = env -i \ + HOME=/dummy \ + NIX_CONF_DIR=/dummy \ + NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ + NIX_STATE_DIR=/dummy + +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @@ -45,11 +52,11 @@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix - $(trace-gen) $(bindir)/nix __dump-args > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @@ -58,7 +65,7 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix - $(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp mv $@.tmp $@ # Generate the HTML manual. From 13c557fe823549db75c3dc24c99c46e1c4e1378e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 11:20:03 +0100 Subject: [PATCH 155/227] fix the hash rewriting for ca-derivations --- src/libstore/build/derivation-goal.cc | 14 ++++++++++++++ tests/content-addressed.nix | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf2bad62c..0e4504857 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3121,6 +3121,20 @@ void DerivationGoal::registerOutputs() newInfo0.references = refs.second; if (refs.first) newInfo0.references.insert(newInfo0.path); + if (scratchPath != newInfo0.path) { + // Also rewrite the output path + auto source = sinkToSource([&](Sink & nextSink) { + StringSink sink; + dumpPath(actualPath, sink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + } assert(newInfo0.ca); return newInfo0; diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 8ca96d4bf..985220f48 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -16,14 +16,16 @@ rec { }; rootCA = mkDerivation { name = "rootCA"; - outputs = [ "out" "dev" ]; + outputs = [ "out" "dev" "foo"]; buildCommand = '' echo "building a CA derivation" echo "The seed is ${toString seed}" mkdir -p $out echo ${rootLegacy}/hello > $out/dep - # test symlink at root + ln -s $out $out/self + # test symlinks at root ln -s $out $dev + ln -s $out $foo ''; __contentAddressed = true; outputHashMode = "recursive"; @@ -34,7 +36,8 @@ rec { buildCommand = '' echo "building a dependent derivation" mkdir -p $out - echo ${rootCA}/hello > $out/dep + cat ${rootCA}/self/dep + echo ${rootCA}/self/dep > $out/dep ''; __contentAddressed = true; outputHashMode = "recursive"; From 0287f8305790a87b128ce09d7d3fa0de7104673c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 12:34:43 +0100 Subject: [PATCH 156/227] Ask for confirmation before allowing flake Nix configuration settings --- src/libexpr/flake/flake.cc | 69 ++++++++++++++++++++----------------- src/libexpr/flake/flake.hh | 2 +- src/libmain/progress-bar.cc | 11 ++++++ src/libutil/logging.hh | 3 ++ 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bdcf63c21..453d219dc 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -233,28 +233,28 @@ static Flake getFlake( if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); - for (auto & option : *nixConfig->value->attrs) { - forceTrivialValue(state, *option.value, *option.pos); - if (option.value->type == tString) - flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)}); - else if (option.value->type == tInt) - flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)}); - else if (option.value->type == tBool) - flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)}); - else if (option.value->isList()) { + for (auto & setting : *nixConfig->value->attrs) { + forceTrivialValue(state, *setting.value, *setting.pos); + if (setting.value->type == tString) + flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); + else if (setting.value->type == tInt) + flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); + else if (setting.value->type == tBool) + flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); + else if (setting.value->isList()) { std::vector ss; - for (unsigned int n = 0; n < option.value->listSize(); ++n) { - auto elem = option.value->listElems()[n]; + for (unsigned int n = 0; n < setting.value->listSize(); ++n) { + auto elem = setting.value->listElems()[n]; if (elem->type != tString) - throw TypeError("list element in flake configuration option '%s' is %s while a string is expected", - option.name, showType(*option.value)); - ss.push_back(state.forceStringNoCtx(*elem, *option.pos)); + throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", + setting.name, showType(*setting.value)); + ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); } - flake.config.options.insert({option.name, ss}); + flake.config.settings.insert({setting.name, ss}); } else - throw TypeError("flake configuration option '%s' is %s", - option.name, showType(*option.value)); + throw TypeError("flake configuration setting '%s' is %s", + setting.name, showType(*setting.value)); } } @@ -637,27 +637,34 @@ Flake::~Flake() { } void ConfigFile::apply() { - for (auto & [name, value] : options) { - // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU. - if (name != "bash-prompt-suffix" && - name != "bash-prompt" && - name != "substituters" && - name != "extra-substituters") - { - warn("ignoring untrusted flake configuration option '%s'", name); - continue; - } + std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; + + for (auto & [name, value] : settings) { + + auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; + // FIXME: Move into libutil/config.cc. + std::string valueS; if (auto s = std::get_if(&value)) - globalConfig.set(name, *s); + valueS = *s; else if (auto n = std::get_if(&value)) - globalConfig.set(name, fmt("%d", n)); + valueS = fmt("%d", n); else if (auto b = std::get_if>(&value)) - globalConfig.set(name, b->t ? "true" : "false"); + valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil + valueS = concatStringsSep(" ", *ss); // FIXME: evil else assert(false); + + if (!whitelist.count(baseName)) { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') { + warn("ignoring untrusted flake configuration setting '%s'", name); + continue; + } + } + + globalConfig.set(name, valueS); } } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 7eebd9044..65ed1ad0a 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -51,7 +51,7 @@ struct ConfigFile { using ConfigValue = std::variant, std::vector>; - std::map options; + std::map settings; void apply(); }; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 07b45b3b5..0e5432fca 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -466,6 +466,17 @@ public: Logger::writeToStdout(s); } } + + std::optional ask(std::string_view msg) override + { + auto state(state_.lock()); + if (!state->active || !isatty(STDIN_FILENO)) return {}; + std::cerr << fmt("\r\e[K%s ", msg); + auto s = trim(readLine(STDIN_FILENO)); + if (s.size() != 1) return {}; + draw(*state); + return s[0]; + } }; Logger * makeProgressBar(bool printBuildLogs) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index e3fe613e8..cd0cb64c5 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -106,6 +106,9 @@ public: formatHelper(f, args...); writeToStdout(f.str()); } + + virtual std::optional ask(std::string_view s) + { return {}; } }; ActivityId getCurActivity(); From 9a586e34ac3ae37bfd18f4e82af26df938ab9d96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 13:11:07 +0100 Subject: [PATCH 157/227] Record trusted/untrusted settings in ~/.local/share/nix --- src/libexpr/flake/flake.cc | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 453d219dc..0eadd94db 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,6 +6,8 @@ #include "fetchers.hh" #include "finally.hh" +#include + namespace nix { using namespace flake; @@ -635,6 +637,27 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } +// setting name -> setting value -> allow or ignore. +typedef std::map> TrustedList; + +Path trustedListPath() +{ + return getDataDir() + "/nix/trusted-settings.json"; +} + +static TrustedList readTrustedList() +{ + auto path = trustedListPath(); + if (!pathExists(path)) return {}; + auto json = nlohmann::json::parse(readFile(path)); + return json; +} + +static void writeTrustedList(const TrustedList & trustedList) +{ + writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); +} + void ConfigFile::apply() { std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; @@ -657,8 +680,28 @@ void ConfigFile::apply() assert(false); if (!whitelist.count(baseName)) { - // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') { + auto trustedList = readTrustedList(); + + bool trusted = false; + + if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + trusted = *saved; + } else { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { + if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = false; + writeTrustedList(trustedList); + } + } else { + if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = trusted = true; + writeTrustedList(trustedList); + } + } + } + + if (!trusted) { warn("ignoring untrusted flake configuration setting '%s'", name); continue; } From 8252a44e96619b6df0901a1f62e60b3d5951fd12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 13:16:36 +0100 Subject: [PATCH 158/227] Move to separate file --- src/libexpr/flake/config.cc | 81 +++++++++++++++++++++++++++++++++++++ src/libexpr/flake/flake.cc | 76 ---------------------------------- 2 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 src/libexpr/flake/config.cc diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc new file mode 100644 index 000000000..63566131e --- /dev/null +++ b/src/libexpr/flake/config.cc @@ -0,0 +1,81 @@ +#include "flake.hh" + +#include + +namespace nix::flake { + +// setting name -> setting value -> allow or ignore. +typedef std::map> TrustedList; + +Path trustedListPath() +{ + return getDataDir() + "/nix/trusted-settings.json"; +} + +static TrustedList readTrustedList() +{ + auto path = trustedListPath(); + if (!pathExists(path)) return {}; + auto json = nlohmann::json::parse(readFile(path)); + return json; +} + +static void writeTrustedList(const TrustedList & trustedList) +{ + writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); +} + +void ConfigFile::apply() +{ + std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; + + for (auto & [name, value] : settings) { + + auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; + + // FIXME: Move into libutil/config.cc. + std::string valueS; + if (auto s = std::get_if(&value)) + valueS = *s; + else if (auto n = std::get_if(&value)) + valueS = fmt("%d", n); + else if (auto b = std::get_if>(&value)) + valueS = b->t ? "true" : "false"; + else if (auto ss = std::get_if>(&value)) + valueS = concatStringsSep(" ", *ss); // FIXME: evil + else + assert(false); + + if (!whitelist.count(baseName)) { + auto trustedList = readTrustedList(); + + bool trusted = false; + + if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + trusted = *saved; + } else { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { + if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = false; + writeTrustedList(trustedList); + } + } else { + if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = trusted = true; + writeTrustedList(trustedList); + } + } + } + + if (!trusted) { + warn("ignoring untrusted flake configuration setting '%s'", name); + continue; + } + } + + globalConfig.set(name, valueS); + } +} + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0eadd94db..3e866e1f9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,8 +6,6 @@ #include "fetchers.hh" #include "finally.hh" -#include - namespace nix { using namespace flake; @@ -637,78 +635,4 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } -// setting name -> setting value -> allow or ignore. -typedef std::map> TrustedList; - -Path trustedListPath() -{ - return getDataDir() + "/nix/trusted-settings.json"; -} - -static TrustedList readTrustedList() -{ - auto path = trustedListPath(); - if (!pathExists(path)) return {}; - auto json = nlohmann::json::parse(readFile(path)); - return json; -} - -static void writeTrustedList(const TrustedList & trustedList) -{ - writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); -} - -void ConfigFile::apply() -{ - std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; - - for (auto & [name, value] : settings) { - - auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; - - // FIXME: Move into libutil/config.cc. - std::string valueS; - if (auto s = std::get_if(&value)) - valueS = *s; - else if (auto n = std::get_if(&value)) - valueS = fmt("%d", n); - else if (auto b = std::get_if>(&value)) - valueS = b->t ? "true" : "false"; - else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil - else - assert(false); - - if (!whitelist.count(baseName)) { - auto trustedList = readTrustedList(); - - bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { - trusted = *saved; - } else { - // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } - } - } - - if (!trusted) { - warn("ignoring untrusted flake configuration setting '%s'", name); - continue; - } - } - - globalConfig.set(name, valueS); - } -} - } From 1fd13d67e85b8365baed1cfb435870e24a7e6979 Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Thu, 26 Nov 2020 14:26:57 +0100 Subject: [PATCH 159/227] archive: disable preallocate-contents by default using fallocate() to preallocate files space does more harm than good: - breaks compression on btrfs - has been called "not the right thing to do" by xfs developers (because delayed allocation that most filesystems implement leads to smarter allocation than what the filesystem needs to do if we upfront fallocate files) --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f1479329f..03534abc4 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,7 +27,7 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, true, "preallocate-contents", + Setting preallocateContents{this, false, "preallocate-contents", "Whether to preallocate files when writing objects with known size."}; }; From 05d9442f68ba906ae50c12deab63fc1b9836b149 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 21:45:28 +0100 Subject: [PATCH 160/227] builtins.fetchGit: Fix shortRev attribute for dirty trees --- src/libexpr/primops/fetchTree.cc | 2 +- tests/fetchGit.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6f637a43..d094edf92 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -39,7 +39,7 @@ void emitTreeAttrs( // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev auto emptyHash = Hash(htSHA1); mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); } if (input.getType() == "git") diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index cedd796f7..76390fa59 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -59,6 +59,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath [[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]] # Fetching with a explicit hash should succeed. path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -132,6 +133,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).shortRev") = 0000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a From 9bd8184f1fb2e91ac4fb7207abe56f2a30a81d97 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 18:20:35 +0100 Subject: [PATCH 161/227] Allow fixed-output derivations to depend on (floating) content-addressed ones Fix an overlook of https://github.com/NixOS/nix/pull/4056 --- src/libstore/build/derivation-goal.cc | 4 ++-- tests/content-addressed.nix | 11 +++++++++++ tests/content-addressed.sh | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 0e4504857..76c49f92c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && - fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 985220f48..61079176f 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -63,4 +63,15 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentFixedOutput = mkDerivation { + name = "dependent-fixed-output"; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA="; + buildCommand = '' + cat ${dependentCA}/dep + echo foo > $out + ''; + + }; } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index bdab09c86..52f7529b5 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -40,6 +40,7 @@ testCutoff () { #testDerivation dependentCA testCutoffFor transitivelyDependentCA testCutoffFor dependentNonCA + testCutoffFor dependentFixedOutput } testGC () { From e224c16d28be738b260d9d15d0503da3febf99de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 14:14:11 +0100 Subject: [PATCH 162/227] Macro hygiene --- src/libutil/logging.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index cd0cb64c5..ff17056f4 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -178,8 +178,8 @@ extern Verbosity verbosity; /* suppress msgs > this */ lightweight status messages. */ #define logErrorInfo(level, errorInfo...) \ do { \ - if (level <= nix::verbosity) { \ - logger->logEI(level, errorInfo); \ + if ((level) <= nix::verbosity) { \ + logger->logEI((level), errorInfo); \ } \ } while (0) From 59276244738ebec32ed0593ee4af615852ff564d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 14:14:57 +0100 Subject: [PATCH 163/227] Lower verbosity for 'Failed to find a machine' message --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 9a0e9f08d..8348d8c91 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -201,7 +201,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep(", ", m.mandatoryFeatures); } - logErrorInfo(lvlInfo, { + logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, { .name = "Remote build", .description = "Failed to find a machine for remote build!", .hint = hint From 3b7e00ce2215b742d9fdb1b8d4a4d76d349028c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Nov 2020 14:36:15 +0100 Subject: [PATCH 164/227] Move primeCache() to Worker::run() We need the missing path info to communicate the worker's remaining goals to the progress bar. --- src/libstore/build/derivation-goal.hh | 12 +----------- src/libstore/build/local-store-build.cc | 22 ++-------------------- src/libstore/build/substitution-goal.hh | 8 +------- src/libstore/build/worker.cc | 21 ++++++++++++++++++++- 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 4976207e0..8ee0be9e1 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -40,9 +40,8 @@ struct InitialOutput { std::optional known; }; -class DerivationGoal : public Goal +struct DerivationGoal : public Goal { -private: /* Whether to use an on-disk .drv file. */ bool useDerivation; @@ -246,7 +245,6 @@ private: friend struct RestrictedStore; -public: DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -264,17 +262,11 @@ public: void work() override; - StorePath getDrvPath() - { - return drvPath; - } - /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); BuildResult getResult() { return result; } -private: /* The states. */ void getDerivation(); void loadDerivation(); @@ -318,8 +310,6 @@ private: /* Run the builder's process. */ void runChild(); - friend int childEntry(void *); - /* Check that the derivation outputs all exist and register them as valid. */ void registerOutputs(); diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc index a05fb5805..c91cda2fd 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/local-store-build.cc @@ -5,25 +5,10 @@ namespace nix { -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); - primeCache(*this, drvPaths); - Goals goals; for (auto & path : drvPaths) { if (path.path.isDerivation()) @@ -44,9 +29,8 @@ void LocalStore::buildPaths(const std::vector & drvPaths, ex = i->ex; } if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); + if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -84,8 +68,6 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {{path}}); - Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); Goals goals = {goal}; diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3ae9a9e6b..dee2cecbf 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,11 +8,8 @@ namespace nix { class Worker; -class SubstitutionGoal : public Goal +struct SubstitutionGoal : public Goal { - friend class Worker; - -private: /* The store path that should be realised through a substitute. */ StorePath storePath; @@ -56,7 +53,6 @@ private: /* Content address for recomputing store path */ std::optional ca; -public: SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); @@ -82,8 +78,6 @@ public: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 17c10cd71..1f8999a4b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -207,7 +207,26 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - for (auto & i : _topGoals) topGoals.insert(i); + std::vector topPaths; + + for (auto & i : _topGoals) { + topGoals.insert(i); + if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + } else if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->storePath}); + } + } + + /* Call queryMissing() efficiently query substitutes. */ + StorePathSet willBuild, willSubstitute, unknown; + uint64_t downloadSize, narSize; + store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); debug("entered goal loop"); From c0d1354b7da9ffc2923bc102abb67d03b655fbbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Nov 2020 12:17:07 +0100 Subject: [PATCH 165/227] Macro hygiene --- src/libutil/logging.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index ff17056f4..82ba54051 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -191,8 +191,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ arguments are evaluated lazily. */ #define printMsg(level, args...) \ do { \ - if (level <= nix::verbosity) { \ - logger->log(level, fmt(args)); \ + auto __lvl = level; \ + if (__lvl <= nix::verbosity) { \ + logger->log(__lvl, fmt(args)); \ } \ } while (0) From 88798613ee288c7a801dcc1e73723a10a385df38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Nov 2020 14:59:03 +0100 Subject: [PATCH 166/227] replaceStrings(): Use std::string_view --- src/libutil/util.cc | 6 +++--- src/libutil/util.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 01ab9111f..c1b12e725 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1273,11 +1273,11 @@ string trim(const string & s, const string & whitespace) } -string replaceStrings(const std::string & s, +string replaceStrings(std::string_view s, const std::string & from, const std::string & to) { - if (from.empty()) return s; - string res = s; + string res(s); + if (from.empty()) return res; size_t pos = 0; while ((pos = res.find(from, pos)) != std::string::npos) { res.replace(pos, from.size(), to); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index cafe93702..117fe86e7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -383,7 +383,7 @@ string trim(const string & s, const string & whitespace = " \n\r\t"); /* Replace all occurrences of a string inside another string. */ -string replaceStrings(const std::string & s, +string replaceStrings(std::string_view s, const std::string & from, const std::string & to); From 438977731cf049cf47873d5825456d47a1aac541 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 1 Dec 2020 14:57:56 +0100 Subject: [PATCH 167/227] shut up clang warnings - Fix some class/struct discrepancies - Explicit the overloading of `run` in the `Cmd*` classes - Ignore a warning in the generated lexer --- src/libexpr/lexer.l | 2 ++ src/libstore/build/goal.hh | 2 +- src/libstore/build/worker.hh | 4 ++-- src/libstore/local-store.hh | 4 ++-- src/nix/run.cc | 8 ++++++++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index f6e83926b..225ab3287 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -12,6 +12,8 @@ %{ +#pragma clang diagnostic ignored "-Wunneeded-internal-declaration" + #include #include "nixexpr.hh" diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0781a9d38..fca4f2d00 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -7,7 +7,7 @@ namespace nix { /* Forward definition. */ struct Goal; -struct Worker; +class Worker; /* A pointer to a goal. */ typedef std::shared_ptr GoalPtr; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 3a53a8def..bf8cc4586 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -8,8 +8,8 @@ namespace nix { /* Forward definition. */ -class DerivationGoal; -class SubstitutionGoal; +struct DerivationGoal; +struct SubstitutionGoal; /* Workaround for not being able to declare a something like diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d4435220d..58ec93f27 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -296,8 +296,8 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend class DerivationGoal; - friend class SubstitutionGoal; + friend struct DerivationGoal; + friend struct SubstitutionGoal; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index 790784382..92a52c6cd 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -22,6 +22,9 @@ std::string chrootHelperName = "__run_in_chroot"; struct RunCommon : virtual Command { + + using Command::run; + void runProgram(ref store, const std::string & program, const Strings & args) @@ -59,6 +62,9 @@ struct RunCommon : virtual Command struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { + + using InstallablesCommand::run; + std::vector command = { getEnv("SHELL").value_or("bash") }; CmdShell() @@ -144,6 +150,8 @@ static auto rCmdShell = registerCommand("shell"); struct CmdRun : InstallableCommand, RunCommon { + using InstallableCommand::run; + std::vector args; CmdRun() From aa684861127eabc18a0d7386a66c5b75d4962897 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 12:43:52 +0100 Subject: [PATCH 168/227] writeFull/writeFile: Use std::string_view --- src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c1b12e725..98a069c3e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -320,7 +320,7 @@ void readFile(const Path & path, Sink & sink) } -void writeFile(const Path & path, const string & s, mode_t mode) +void writeFile(const Path & path, std::string_view s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -663,7 +663,7 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr } -void writeFull(int fd, const string & s, bool allowInterrupts) +void writeFull(int fd, std::string_view s, bool allowInterrupts) { writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 117fe86e7..eb872dd4b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -106,7 +106,7 @@ string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s, mode_t mode = 0666); +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); void writeFile(const Path & path, Source & source, mode_t mode = 0666); @@ -157,7 +157,7 @@ void replaceSymlink(const Path & target, const Path & link, requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); -void writeFull(int fd, const string & s, bool allowInterrupts = true); +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); From faa31f40846f7a4dbc2487d000b112a6aef69d1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:00:43 +0100 Subject: [PATCH 169/227] Sink: Use std::string_view --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/build/derivation-goal.cc | 8 +- src/libstore/daemon.cc | 4 +- src/libstore/filetransfer.cc | 18 ++-- src/libstore/filetransfer.hh | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/references.cc | 50 +++++------- src/libstore/references.hh | 4 +- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/store-api.cc | 4 +- src/libutil/archive.cc | 18 ++-- src/libutil/archive.hh | 8 +- src/libutil/compression.cc | 113 +++++++++++++------------- src/libutil/hash.cc | 18 ++-- src/libutil/hash.hh | 2 +- src/libutil/serialise.cc | 48 +++++------ src/libutil/serialise.hh | 52 +++++------- src/libutil/util.cc | 30 +++---- src/libutil/util.hh | 1 - src/nix/make-content-addressable.cc | 4 +- 21 files changed, 182 insertions(+), 211 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f6224d6a0..a918b7208 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) promise.set_exception(std::current_exception()); } }}); - auto data = promise.get_future().get(); - sink((unsigned char *) data->data(), data->size()); + sink(*promise.get_future().get()); } std::shared_ptr BinaryCacheStore::getFile(const std::string & path) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 76c49f92c..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -916,10 +916,8 @@ void DerivationGoal::buildDone() LogSink(Activity & act) : act(act) { } - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - + void operator() (std::string_view data) override { + for (auto c : data) { if (c == '\n') { flushLine(); } else { @@ -3127,7 +3125,7 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 60cca4fda..cb214bee3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -153,10 +153,10 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) + void operator () (std::string_view data) { to << STDERR_WRITE; - writeString(data, len, to); + writeString(data, to); } }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c2c65af05..31b4215a9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) - , finalSink([this](const unsigned char * data, size_t len) { + , finalSink([this](std::string_view data) { if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); /* Only write data to the sink if this is a successful response. */ if (successfulStatuses.count(httpStatus)) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + writtenToSink += data.size(); + this->request.dataCallback(data); } } else - this->result.data->append((char *) data, len); + this->result.data->append(data); }) { if (!request.expectedETag.empty()) @@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer } if (errorSink) - (*errorSink)((unsigned char *) contents, realSize); - (*decompressionSink)((unsigned char *) contents, realSize); + (*errorSink)({(char *) contents, realSize}); + (*decompressionSink)({(char *) contents, realSize}); return realSize; } catch (...) { @@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) state->request.notify_one(); }); - request.dataCallback = [_state](char * buf, size_t len) { + request.dataCallback = [_state](std::string_view data) { auto state(_state->lock()); @@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) /* Append data to the buffer and wake up the calling thread. */ - state->data.append(buf, len); + state->data.append(data); state->avail.notify_one(); }; @@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) if it's blocked on a full buffer. We don't hold the state lock while doing this to prevent blocking the download thread if sink() takes a long time. */ - sink((unsigned char *) chunk.data(), chunk.size()); + sink(chunk); } } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index c89c51a21..afc7e7aa6 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -61,7 +61,7 @@ struct FileTransferRequest bool decompress = true; std::shared_ptr data; std::string mimeType; - std::function dataCallback; + std::function dataCallback; FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index a9efdd0b6..e35031f7a 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor parents.top()->start = pos; } - void receiveContents(unsigned char * data, size_t len) override + void receiveContents(std::string_view data) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libstore/references.cc b/src/libstore/references.cc index d2096cb49..eb117b5ba 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -55,27 +55,23 @@ struct RefScanSink : Sink RefScanSink() { } - void operator () (const unsigned char * data, size_t len); + void operator () (std::string_view data) override + { + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + std::string(data, 0, refLength); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search((const unsigned char *) data.data(), data.size(), hashes, seen); + + size_t tailLen = data.size() <= refLength ? data.size() : refLength; + tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)); + tail.append({data.data() + data.size() - tailLen, tailLen}); + } }; -void RefScanSink::operator () (const unsigned char * data, size_t len) -{ - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + string((const char *) data, len > refLength ? refLength : len); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search(data, len, hashes, seen); - - size_t tailLen = len <= refLength ? len : refLength; - tail = - string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + - string((const char *) data + len - tailLen, tailLen); -} - - std::pair scanForReferences(const string & path, const PathSet & refs) { @@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S assert(from.size() == to.size()); } -void RewritingSink::operator () (const unsigned char * data, size_t len) +void RewritingSink::operator () (std::string_view data) { std::string s(prev); - s.append((const char *) data, len); + s.append(data); size_t j = 0; while ((j = s.find(from, j)) != string::npos) { @@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len) pos += consumed; - if (consumed) nextSink((unsigned char *) s.data(), consumed); + if (consumed) nextSink(s.substr(0, consumed)); } void RewritingSink::flush() { if (prev.empty()) return; pos += prev.size(); - nextSink((unsigned char *) prev.data(), prev.size()); + nextSink(prev); prev.clear(); } @@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) { } -void HashModuloSink::operator () (const unsigned char * data, size_t len) +void HashModuloSink::operator () (std::string_view data) { - rewritingSink(data, len); + rewritingSink(data); } HashResult HashModuloSink::finish() @@ -176,10 +172,8 @@ HashResult HashModuloSink::finish() NAR with self-references and a NAR with some of the self-references already zeroed out do not produce a hash collision. FIXME: proof. */ - for (auto & pos : rewritingSink.matches) { - auto s = fmt("|%d", pos); - hashSink((unsigned char *) s.data(), s.size()); - } + for (auto & pos : rewritingSink.matches) + hashSink(fmt("|%d", pos)); auto h = hashSink.finish(); return {h.first, rewritingSink.pos}; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c2efd095c..4f12e6b21 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -19,7 +19,7 @@ struct RewritingSink : Sink RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; void flush(); }; @@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink HashModuloSink(HashType ht, const std::string & modulus); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; HashResult finish() override; }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48af9f0c4..6e0c85237 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -857,7 +857,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (!source) throw Error("no source"); size_t len = readNum(from); auto buf = std::make_unique(len); - writeString(buf.get(), source->read(buf.get(), len), to); + writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 552c4aac7..4519dd5b5 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -398,7 +398,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, res.data->size(), res.durationMs); - sink((unsigned char *) res.data->data(), res.data->size()); + sink(*res.data); } else throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3b2be05cb..27be66cac 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -772,8 +772,8 @@ void copyStorePath(ref srcStore, ref dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink progressSink([&](const unsigned char * data, size_t len) { - total += len; + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); act.progress(total, info->narSize); }); TeeSink tee { sink, progressSink }; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 03534abc4..046ef841b 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -57,7 +57,7 @@ static void dumpContents(const Path & path, size_t size, auto n = std::min(left, buf.size()); readFull(fd.get(), buf.data(), n); left -= n; - sink(buf.data(), n); + sink({(char *) buf.data(), n}); } writePadding(size, sink); @@ -162,7 +162,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents(buf.data(), n); + sink.receiveContents({(char *) buf.data(), n}); left -= n; } @@ -300,21 +300,21 @@ struct RestoreSink : ParseSink Path dstPath; AutoCloseFD fd; - void createDirectory(const Path & path) + void createDirectory(const Path & path) override { Path p = dstPath + path; if (mkdir(p.c_str(), 0777) == -1) throw SysError("creating directory '%1%'", p); }; - void createRegularFile(const Path & path) + void createRegularFile(const Path & path) override { Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); if (!fd) throw SysError("creating file '%1%'", p); } - void isExecutable() + void isExecutable() override { struct stat st; if (fstat(fd.get(), &st) == -1) @@ -323,7 +323,7 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(uint64_t len) + void preallocateContents(uint64_t len) override { if (!archiveSettings.preallocateContents) return; @@ -341,12 +341,12 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, size_t len) + void receiveContents(std::string_view data) override { - writeFull(fd.get(), data, len); + writeFull(fd.get(), data); } - void createSymlink(const Path & path, const string & target) + void createSymlink(const Path & path, const string & target) override { Path p = dstPath + path; nix::createSymlink(target, p); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 5665732d2..fe22435ef 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -58,7 +58,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(unsigned char * data, size_t len) { }; + virtual void receiveContents(std::string_view data) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; @@ -72,14 +72,14 @@ struct RetrieveRegularNARSink : ParseSink RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - void createDirectory(const Path & path) + void createDirectory(const Path & path) override { regular = false; } - void receiveContents(unsigned char * data, size_t len) + void receiveContents(std::string_view data) override { - sink(data, len); + sink(data); } void createSymlink(const Path & path, const string & target) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a117ddc72..986ba2976 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -22,18 +22,17 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; - while (len) { - size_t n = std::min(CHUNK_SIZE, len); - writeInternal(data, n); - data += n; - len -= n; + while (!data.empty()) { + size_t n = std::min(CHUNK_SIZE, data.size()); + writeInternal(data); + data.remove_prefix(n); } } - virtual void writeInternal(const unsigned char * data, size_t len) = 0; + virtual void writeInternal(std::string_view data) = 0; }; struct NoneSink : CompressionSink @@ -41,7 +40,7 @@ struct NoneSink : CompressionSink Sink & nextSink; NoneSink(Sink & nextSink) : nextSink(nextSink) { } void finish() override { flush(); } - void write(const unsigned char * data, size_t len) override { nextSink(data, len); } + void write(std::string_view data) override { nextSink(data); } }; struct GzipDecompressionSink : CompressionSink @@ -75,28 +74,28 @@ struct GzipDecompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (Bytef *) data; - strm.avail_in = len; + strm.next_in = (Bytef *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); int ret = inflate(&strm,Z_SYNC_FLUSH); if (ret != Z_OK && ret != Z_STREAM_END) throw CompressionError("error while decompressing gzip file: %d (%d, %d)", - zError(ret), len, strm.avail_in); + zError(ret), data.size(), strm.avail_in); finished = ret == Z_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (Bytef *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -130,25 +129,25 @@ struct XzDecompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - strm.next_in = data; - strm.avail_in = len; + strm.next_in = (const unsigned char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); if (ret != LZMA_OK && ret != LZMA_STREAM_END) throw CompressionError("error %d while decompressing xz file", ret); finished = ret == LZMA_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -181,15 +180,15 @@ struct BzipDecompressionSink : ChunkedCompressionSink void finish() override { flush(); - write(nullptr, 0); + write({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char *) data.data(); + strm.avail_in = data.size(); while (strm.avail_in) { checkInterrupt(); @@ -201,7 +200,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink finished = ret == BZ_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -230,17 +229,17 @@ struct BrotliDecompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - const uint8_t * next_in = data; - size_t avail_in = len; + auto next_in = (const uint8_t *) data.data(); + size_t avail_in = data.size(); uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); - while (!finished && (!data || avail_in)) { + while (!finished && (!data.data() || avail_in)) { checkInterrupt(); if (!BrotliDecoderDecompressStream(state, @@ -250,7 +249,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink throw CompressionError("error while decompressing brotli file"); if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - avail_out}); next_out = outbuf; avail_out = sizeof(outbuf); } @@ -338,25 +337,25 @@ struct XzCompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - strm.next_in = data; - strm.avail_in = len; + strm.next_in = (const unsigned char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); if (ret != LZMA_OK && ret != LZMA_STREAM_END) throw CompressionError("error %d while compressing xz file", ret); finished = ret == LZMA_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -389,27 +388,27 @@ struct BzipCompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH); if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) throw CompressionError("error %d while compressing bzip2 file", ret); finished = ret == BZ_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -439,28 +438,28 @@ struct BrotliCompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - const uint8_t * next_in = data; - size_t avail_in = len; + auto next_in = (const uint8_t *) data.data(); + size_t avail_in = data.size(); uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); - while (!finished && (!data || avail_in)) { + while (!finished && (!data.data() || avail_in)) { checkInterrupt(); if (!BrotliEncoderCompressStream(state, - data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in, &avail_out, &next_out, nullptr)) throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out}); next_out = outbuf; avail_out = sizeof(outbuf); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 8efff190a..4df8b4ecb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -291,12 +291,12 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, size_t len) + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); + if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } @@ -314,7 +314,7 @@ Hash hashString(HashType ht, std::string_view s) Ctx ctx; Hash hash(ht); start(ht, ctx); - update(ht, ctx, (const unsigned char *) s.data(), s.length()); + update(ht, ctx, s); finish(ht, ctx, hash.hash); return hash; } @@ -341,10 +341,10 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(const unsigned char * data, size_t len) +void HashSink::write(std::string_view data) { - bytes += len; - update(ht, *ctx, data, len); + bytes += data.size(); + update(ht, *ctx, data); } HashResult HashSink::finish() diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 6d6eb70ca..1b626dd85 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -156,7 +156,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len) override; + void write(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 038ede049..534ad5cbb 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -11,23 +11,23 @@ namespace nix { -void BufferedSink::operator () (const unsigned char * data, size_t len) +void BufferedSink::operator () (std::string_view data) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); - while (len) { + while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + len >= bufSize) { + if (bufPos + data.size() >= bufSize) { flush(); - write(data, len); + write(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; - memcpy(buffer.get() + bufPos, data, n); - data += n; bufPos += n; len -= n; + size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + memcpy(buffer.get() + bufPos, data.data(), n); + data.remove_prefix(n); bufPos += n; if (bufPos == bufSize) flush(); } } @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write(buffer.get(), n); + write({buffer.get(), n}); } @@ -59,9 +59,9 @@ static void warnLargeDump() } -void FdSink::write(const unsigned char * data, size_t len) +void FdSink::write(std::string_view data) { - written += len; + written += data.size(); static bool warned = false; if (warn && !warned) { if (written > threshold) { @@ -70,7 +70,7 @@ void FdSink::write(const unsigned char * data, size_t len) } } try { - writeFull(fd, data, len); + writeFull(fd, data); } catch (SysError & e) { _good = false; throw; @@ -101,7 +101,7 @@ void Source::drainInto(Sink & sink) size_t n; try { n = read(buf.data(), buf.size()); - sink(buf.data(), n); + sink({(char *) buf.data(), n}); } catch (EndOfFile &) { break; } @@ -229,9 +229,9 @@ std::unique_ptr sinkToSource( { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) yield(std::string(data)); + }); fun(sink); }); @@ -260,22 +260,22 @@ void writePadding(size_t len, Sink & sink) if (len % 8) { unsigned char zero[8]; memset(zero, 0, sizeof(zero)); - sink(zero, 8 - (len % 8)); + sink({(char *) zero, 8 - (len % 8)}); } } -void writeString(const unsigned char * buf, size_t len, Sink & sink) +void writeString(std::string_view data, Sink & sink) { - sink << len; - sink(buf, len); - writePadding(len, sink); + sink << data.size(); + sink(data); + writePadding(data.size(), sink); } Sink & operator << (Sink & sink, const string & s) { - writeString((const unsigned char *) s.data(), s.size(), sink); + writeString(s, sink); return sink; } @@ -394,14 +394,14 @@ Error readError(Source & source) } -void StringSink::operator () (const unsigned char * data, size_t len) +void StringSink::operator () (std::string_view data) { static bool warned = false; if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s->append((const char *) data, len); + s->append(data); } size_t ChainSource::read(unsigned char * data, size_t len) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5c7d3ce76..f9ab79997 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -14,19 +14,14 @@ namespace nix { struct Sink { virtual ~Sink() { } - virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual void operator () (std::string_view data) = 0; virtual bool good() { return true; } - - void operator () (const std::string & s) - { - (*this)((const unsigned char *) s.data(), s.size()); - } }; /* Just throws away data. */ struct NullSink : Sink { - void operator () (const unsigned char * data, size_t len) override + void operator () (std::string_view data) override { } }; @@ -35,21 +30,16 @@ struct NullSink : Sink struct BufferedSink : virtual Sink { size_t bufSize, bufPos; - std::unique_ptr buffer; + std::unique_ptr buffer; BufferedSink(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPos(0), buffer(nullptr) { } - void operator () (const unsigned char * data, size_t len) override; - - void operator () (const std::string & s) - { - Sink::operator()(s); - } + void operator () (std::string_view data) override; void flush(); - virtual void write(const unsigned char * data, size_t len) = 0; + virtual void write(std::string_view data) = 0; }; @@ -119,7 +109,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(const unsigned char * data, size_t len) override; + void write(std::string_view data) override; bool good() override; @@ -163,7 +153,7 @@ struct StringSink : Sink s->reserve(reservedSize); }; StringSink(ref s) : s(s) { }; - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; }; @@ -182,10 +172,10 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (const unsigned char * data, size_t len) + virtual void operator () (std::string_view data) { - sink1(data, len); - sink2(data, len); + sink1(data); + sink2(data); } }; @@ -200,7 +190,7 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, n); + sink({(char *) data, n}); return n; } }; @@ -241,24 +231,24 @@ struct LengthSink : Sink { uint64_t length = 0; - virtual void operator () (const unsigned char * _, size_t len) + void operator () (std::string_view data) override { - length += len; + length += data.size(); } }; /* Convert a function into a sink. */ struct LambdaSink : Sink { - typedef std::function lambda_t; + typedef std::function lambda_t; lambda_t lambda; LambdaSink(const lambda_t & lambda) : lambda(lambda) { } - virtual void operator () (const unsigned char * data, size_t len) + void operator () (std::string_view data) override { - lambda(data, len); + lambda(data); } }; @@ -302,7 +292,7 @@ std::unique_ptr sinkToSource( void writePadding(size_t len, Sink & sink); -void writeString(const unsigned char * buf, size_t len, Sink & sink); +void writeString(std::string_view s, Sink & sink); inline Sink & operator << (Sink & sink, uint64_t n) { @@ -315,7 +305,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) buf[5] = (n >> 40) & 0xff; buf[6] = (n >> 48) & 0xff; buf[7] = (unsigned char) (n >> 56) & 0xff; - sink(buf, sizeof(buf)); + sink({(char *) buf, sizeof(buf)}); return sink; } @@ -484,7 +474,7 @@ struct FramedSink : nix::BufferedSink } } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -493,8 +483,8 @@ struct FramedSink : nix::BufferedSink ex = nullptr; std::rethrow_exception(ex2); } - to << len; - to(data, len); + to << data.size(); + to(data); }; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 98a069c3e..3d4fc4c25 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -346,7 +346,7 @@ void writeFile(const Path & path, Source & source, mode_t mode) while (true) { try { auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), (unsigned char *) buf.data(), n); + writeFull(fd.get(), {(char *) buf.data(), n}); } catch (EndOfFile &) { break; } } } catch (Error & e) { @@ -648,24 +648,16 @@ void readFull(int fd, unsigned char * buf, size_t count) } -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) -{ - while (count) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, (char *) buf, count); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) { - count -= res; - buf += res; - } - } -} - - void writeFull(int fd, std::string_view s, bool allowInterrupts) { - writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } } @@ -705,7 +697,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("reading from file"); } else if (rd == 0) break; - else sink(buf.data(), rd); + else sink({(char *) buf.data(), (size_t) rd}); } } @@ -1153,7 +1145,7 @@ void runProgram2(const RunOptions & options) } catch (EndOfFile &) { break; } - writeFull(in.writeSide.get(), buf.data(), n); + writeFull(in.writeSide.get(), {(char *) buf.data(), n}); } promise.set_value(); } catch (...) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index eb872dd4b..fcb7cad39 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -156,7 +156,6 @@ void replaceSymlink(const Path & target, const Path & link, /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index df3ec5194..12c2cf776 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -73,7 +73,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON *sink.s = rewriteStrings(*sink.s, rewrites); HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + hashModuloSink(*sink.s); auto narHash = hashModuloSink.finish().first; @@ -94,7 +94,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); From 1b79b5b983a6c775766bd0d1c7881042188998b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:10:56 +0100 Subject: [PATCH 170/227] read(): Use char * instead of unsigned char * This gets rid of some pointless casts. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libutil/archive.cc | 8 +++--- src/libutil/serialise.cc | 34 +++++++++++------------ src/libutil/serialise.hh | 44 +++++++++++++++--------------- src/libutil/tarfile.cc | 2 +- src/libutil/util.cc | 12 ++++---- src/libutil/util.hh | 2 +- 11 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cb214bee3..e5cfe94cb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -165,7 +165,7 @@ struct TunnelSource : BufferedSource Source & from; BufferedSink & to; TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } - size_t readUnbuffered(unsigned char * data, size_t len) override + size_t readUnbuffered(char * data, size_t len) override { to << STDERR_READ << len; to.flush(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 93d073768..348e5d0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1143,7 +1143,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, dump.resize(oldSize + want); auto got = 0; try { - got = source.read((uint8_t *) dump.data() + oldSize, want); + got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index e35031f7a..1427a0f98 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { auto n = source.read(data, len); pos += n; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 2d02a181b..63bde92de 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -75,7 +75,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) throw SysError("seeking in '%s'", cacheFile); std::string buf(length, 0); - readFull(fd.get(), (unsigned char *) buf.data(), length); + readFull(fd.get(), buf.data(), length); return buf; }); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e0c85237..be29f8e6f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -856,7 +856,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * else if (msg == STDERR_READ) { if (!source) throw Error("no source"); size_t len = readNum(from); - auto buf = std::make_unique(len); + auto buf = std::make_unique(len); writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 046ef841b..ed0eb2fb5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -50,14 +50,14 @@ static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(65536); + std::vector buf(65536); size_t left = size; while (left > 0) { auto n = std::min(left, buf.size()); readFull(fd.get(), buf.data(), n); left -= n; - sink({(char *) buf.data(), n}); + sink({buf.data(), n}); } writePadding(size, sink); @@ -155,14 +155,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size); uint64_t left = size; - std::vector buf(65536); + std::vector buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents({(char *) buf.data(), n}); + sink.receiveContents({buf.data(), n}); left -= n; } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 534ad5cbb..87c1099a1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -84,7 +84,7 @@ bool FdSink::good() } -void Source::operator () (unsigned char * data, size_t len) +void Source::operator () (char * data, size_t len) { while (len) { size_t n = read(data, len); @@ -96,12 +96,12 @@ void Source::operator () (unsigned char * data, size_t len) void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::vector buf(8192); while (true) { size_t n; try { n = read(buf.data(), buf.size()); - sink({(char *) buf.data(), n}); + sink({buf.data(), n}); } catch (EndOfFile &) { break; } @@ -117,9 +117,9 @@ std::string Source::drain() } -size_t BufferedSource::read(unsigned char * data, size_t len) +size_t BufferedSource::read(char * data, size_t len) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); @@ -138,12 +138,12 @@ bool BufferedSource::hasData() } -size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(char * data, size_t len) { ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, len); + n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } @@ -158,10 +158,10 @@ bool FdSource::good() } -size_t StringSource::read(unsigned char * data, size_t len) +size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); - size_t n = s.copy((char *) data, len, pos); + size_t n = s.copy(data, len, pos); pos += n; return n; } @@ -225,7 +225,7 @@ std::unique_ptr sinkToSource( std::string cur; size_t pos = 0; - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { @@ -244,7 +244,7 @@ std::unique_ptr sinkToSource( } auto n = std::min(cur.size() - pos, len); - memcpy(data, (unsigned char *) cur.data() + pos, n); + memcpy(data, cur.data() + pos, n); pos += n; return n; @@ -258,9 +258,9 @@ std::unique_ptr sinkToSource( void writePadding(size_t len, Sink & sink) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; memset(zero, 0, sizeof(zero)); - sink({(char *) zero, 8 - (len % 8)}); + sink({zero, 8 - (len % 8)}); } } @@ -321,7 +321,7 @@ Sink & operator << (Sink & sink, const Error & ex) void readPadding(size_t len, Source & source) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) @@ -330,7 +330,7 @@ void readPadding(size_t len, Source & source) } -size_t readString(unsigned char * buf, size_t max, Source & source) +size_t readString(char * buf, size_t max, Source & source) { auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); @@ -345,7 +345,7 @@ string readString(Source & source, size_t max) auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); - source((unsigned char*) res.data(), len); + source(res.data(), len); readPadding(len, source); return res; } @@ -404,7 +404,7 @@ void StringSink::operator () (std::string_view data) s->append(data); } -size_t ChainSource::read(unsigned char * data, size_t len) +size_t ChainSource::read(char * data, size_t len) { if (useSecond) { return source2.read(data, len); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index f9ab79997..5bbbc7ce3 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -51,12 +51,12 @@ struct Source /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. It blocks until all the requested data is available, or throws an error if it is not going to be available. */ - void operator () (unsigned char * data, size_t len); + void operator () (char * data, size_t len); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -71,18 +71,18 @@ struct Source struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; - std::unique_ptr buffer; + std::unique_ptr buffer; BufferedSource(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; bool hasData(); protected: /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + virtual size_t readUnbuffered(char * data, size_t len) = 0; }; @@ -138,7 +138,7 @@ struct FdSource : BufferedSource bool good() override; protected: - size_t readUnbuffered(unsigned char * data, size_t len) override; + size_t readUnbuffered(char * data, size_t len) override; private: bool _good = true; }; @@ -163,7 +163,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; }; @@ -187,10 +187,10 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(unsigned char * data, size_t len) + size_t read(char * data, size_t len) { size_t n = orig.read(data, len); - sink({(char *) data, n}); + sink({data, n}); return n; } }; @@ -202,7 +202,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(unsigned char * data, size_t len) + size_t read(char * data, size_t len) { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -216,7 +216,7 @@ struct SizedSource : Source /* Consume the original source until no remain data is left to consume. */ size_t drainAll() { - std::vector buf(8192); + std::vector buf(8192); size_t sum = 0; while (this->remain > 0) { size_t n = read(buf.data(), buf.size()); @@ -256,13 +256,13 @@ struct LambdaSink : Sink /* Convert a function into a source. */ struct LambdaSource : Source { - typedef std::function lambda_t; + typedef std::function lambda_t; lambda_t lambda; LambdaSource(const lambda_t & lambda) : lambda(lambda) { } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { return lambda(data, len); } @@ -278,7 +278,7 @@ struct ChainSource : Source : source1(s1), source2(s2) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; }; @@ -322,7 +322,7 @@ template T readNum(Source & source) { unsigned char buf[8]; - source(buf, sizeof(buf)); + source((char *) buf, sizeof(buf)); uint64_t n = ((uint64_t) buf[0]) | @@ -354,7 +354,7 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); -size_t readString(unsigned char * buf, size_t max, Source & source); +size_t readString(char * buf, size_t max, Source & source); string readString(Source & source, size_t max = std::numeric_limits::max()); template T readStrings(Source & source); @@ -386,9 +386,9 @@ struct StreamToSourceAdapter : Source : istream(istream) { } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { - if (!istream->read((char *) data, len)) { + if (!istream->read(data, len)) { if (istream->eof()) { if (istream->gcount() == 0) throw EndOfFile("end of file"); @@ -411,7 +411,7 @@ struct FramedSource : Source { Source & from; bool eof = false; - std::vector pending; + std::vector pending; size_t pos = 0; FramedSource(Source & from) : from(from) @@ -423,13 +423,13 @@ struct FramedSource : Source while (true) { auto n = readInt(from); if (!n) break; - std::vector data(n); + std::vector data(n); from(data.data(), n); } } } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { if (eof) throw EndOfFile("reached end of FramedSource"); @@ -439,7 +439,7 @@ struct FramedSource : Source eof = true; return 0; } - pending = std::vector(len); + pending = std::vector(len); pos = 0; from(pending.data(), len); } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index c4d8a4f91..2da169ba7 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -66,7 +66,7 @@ private: *buffer = self->buffer.data(); try { - return self->source->read(self->buffer.data(), 4096); + return self->source->read((char *) self->buffer.data(), 4096); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3d4fc4c25..e6b6d287d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -340,13 +340,13 @@ void writeFile(const Path & path, Source & source, mode_t mode) if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(64 * 1024); + std::vector buf(64 * 1024); try { while (true) { try { auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {(char *) buf.data(), n}); + writeFull(fd.get(), {buf.data(), n}); } catch (EndOfFile &) { break; } } } catch (Error & e) { @@ -632,11 +632,11 @@ void replaceSymlink(const Path & target, const Path & link, } -void readFull(int fd, unsigned char * buf, size_t count) +void readFull(int fd, char * buf, size_t count) { while (count) { checkInterrupt(); - ssize_t res = read(fd, (char *) buf, count); + ssize_t res = read(fd, buf, count); if (res == -1) { if (errno == EINTR) continue; throw SysError("reading from file"); @@ -1137,7 +1137,7 @@ void runProgram2(const RunOptions & options) in.readSide = -1; writerThread = std::thread([&]() { try { - std::vector buf(8 * 1024); + std::vector buf(8 * 1024); while (true) { size_t n; try { @@ -1145,7 +1145,7 @@ void runProgram2(const RunOptions & options) } catch (EndOfFile &) { break; } - writeFull(in.writeSide.get(), {(char *) buf.data(), n}); + writeFull(in.writeSide.get(), {buf.data(), n}); } promise.set_value(); } catch (...) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fcb7cad39..0f82bed78 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -155,7 +155,7 @@ void replaceSymlink(const Path & target, const Path & link, /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ -void readFull(int fd, unsigned char * buf, size_t count); +void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); From a8a96dbaf8ae4f49491903876ddaba44be9d3a39 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Dec 2020 14:23:38 +0100 Subject: [PATCH 171/227] Add forgotten `override` annotation --- src/libutil/archive.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fe22435ef..9e9e11b1a 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -82,7 +82,7 @@ struct RetrieveRegularNARSink : ParseSink sink(data); } - void createSymlink(const Path & path, const string & target) + void createSymlink(const Path & path, const string & target) override { regular = false; } From 0d9e1af695ac238b0f8c8bc296bf01eb1430e26b Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Dec 2020 14:33:20 +0100 Subject: [PATCH 172/227] Remove an `unknown pragma` gcc warning --- src/libexpr/lexer.l | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 225ab3287..7298419d9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -12,7 +12,9 @@ %{ +#ifdef __clang__ #pragma clang diagnostic ignored "-Wunneeded-internal-declaration" +#endif #include From d8fc1bb7b00dd7b13d667d3cb41bfcbe0df699d0 Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 2 Dec 2020 10:15:18 -0500 Subject: [PATCH 173/227] fix tokens documentation --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index eabd83e3f..4655ca058 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -867,7 +867,7 @@ public: Example `~/.config/nix/nix.conf`: ``` - access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk ``` Example `~/code/flake.nix`: From 44da19f73cfa87bb09c14fe64fce97904db0970c Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Wed, 2 Dec 2020 17:00:32 +0100 Subject: [PATCH 174/227] Adds Nix CLI Guideline to docs As we are working towards Nix 3.0 we want to make sure that we make a huge step forward in Nix's user experience. And once 3.0 is out of the door we need to make sure that all future commands and features keep up the standard of user experience. This PR adds a CLI guideline document to the Nix documentation. Consider this document a good starting point and a checklist when somebody will be (re)implementing commands. Clearly this guideline does nothing to improve user experience on its own and can only be useful as long as it is going to be read and cared for. But it is a first step into that direction. --- doc/manual/src/SUMMARY.md | 1 + doc/manual/src/command-ref/cli-guideline.md | 571 ++++++++++++++++++++ 2 files changed, 572 insertions(+) create mode 100644 doc/manual/src/command-ref/cli-guideline.md diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md index 8281f683f..27a08bb43 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md @@ -63,6 +63,7 @@ - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) - [nix](command-ref/nix.md) + - [CLI guideline](command-ref/cli-guideline.md) - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) diff --git a/doc/manual/src/command-ref/cli-guideline.md b/doc/manual/src/command-ref/cli-guideline.md new file mode 100644 index 000000000..01df07136 --- /dev/null +++ b/doc/manual/src/command-ref/cli-guideline.md @@ -0,0 +1,571 @@ +# CLI guideline + +## Goals + +Purpose of this document is to provide a clear direction to **help design +delightful command line** experience. This document contain guidelines to +follow to ensure a consistent and approachable user experience. + +## Overview + +`nix` command provides a single entry to a number of sub-commands that help +**developers and system administrators** in the life-cycle of a software +project. We particularly need to pay special attention to help and assist new +users of Nix. + +# Naming the `COMMANDS` + +Words matter. Naming is an important part of the usability. Users will be +interacting with Nix on a regular basis so we should **name things for ease of +understanding**. + +We recommend following the [Principle of Least +Astonishment](https://en.wikipedia.org/wiki/Principle_of_least_astonishment). +This means that you should **never use acronyms or abbreviations** unless they +are commonly used in other tools (e.g. `nix init`). And if the command name is +too long (> 10-12 characters) then shortening it makes sense (e.g. +“prioritization” → “priority”). + +Commands should **follow a noun-verb dialogue**. Although noun-verb formatting +seems backwards from a speaking perspective (i.e. `nix store copy` vs. `nix +copy store`) it allows us to organize commands the same way users think about +completing an action (the group first, then the command). + +## Naming rules + +Rules are there to guide you by limiting your options. But not everything can +fit the rules all the time. In those cases document the exceptions in [Appendix +1: Commands naming exceptions](#appendix-1-commands-naming-exceptions) and +provide reason. The rules want to force a Nix developer to look, not just at +the command at hand, but also the command in a full context alongside other +`nix` commands. + +```shell +$ nix [] [] [] +``` + +- `GROUP`, `COMMAND`, `ARGUMENTS` and `OPTIONS` should be lowercase and in a + singular form. +- `GROUP` should be a **NOUN**. +- `COMMAND` should be a **VERB**. +- `ARGUMENTS` and `OPTIONS` are discussed in [*Input* section](#input). + +## Classification + +Some commands are more important, some less. While we want all of our commands +to be perfect we can only spend limited amount of time testing and improving +them. + +This classification tries to separate commands in 3 categories in terms of +their importance in regards to the new users. Users who are likely to be +impacted the most by bad user experience. This does not mean that we will not +take care of Tier3 commands, it means we will only put more helpful details for +beginners into Tier1 commands. + +- **Tier1**: Commands used for our main use cases and most likely used by new + users. From Tier1 commands we expect attention to details, such as: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) + and [aligning of text](#text-alignment). + - [Autocomplete](#shell-completion) of options. + - Show [next possible steps](#next-steps). + - Showing some [“tips”](#educate-the-user) when running logs running tasks + (eg. building / downloading) in order to teach users interesting bits of + Nix ecosystem. + - [Help pages](#help-is-essential) to be as good as we can write them + pointing to external documentation and tutorials for more. +- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really + exposing some implementation detail, but not something that we expect a user. + From Tier2 command we expect less attention to details, but still some: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) + and [aligning of text](#text-alignment). + - [Autocomplete](#shell-completion) of options. +- **Tier3**: Commands that expose certain internal functionality of `nix`, + mostly used by other scripts. + - [Autocomplete](#shell-completion) of options. + +# Help is essential + +Help should be built into your command line so that new users can gradually +discover new features when they need them. + +## Looking for help + +Since there is no standard way how user will look for help we rely on ways help +is provided by commonly used tools. As a guide for this we took `git` and +whenever in doubt look at it as a preferred direction. + +The rules are: + +- Help is shown by using `--help` or `help` command (eg `nix` `--``help` or + `nix help`). +- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show + a summary** of most common use cases. Summary is presented on the STDOUT + without any use of PAGER. +- For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the + man page of that command. By default the PAGER is used (as in `git`). +- At the end of either summary or man page there should be an URL pointing to + an online version of more detailed documentation. +- The structure of summaries and man pages should be the same as in `git`. + +## Anticipate where help is needed + +Even better then requiring the user to search for help is to anticipate and +predict when user might need it. Either because the lack of discoverability, +typo in the input or simply taking the opportunity to teach the user of +interesting - but less visible - details. + +### Shell completion + +This type of help is most common and almost expected by users. We need to +**provide the best shell completion** for `bash`, `zsh` and `fish`. + +Completion needs to be **context aware**, this mean when a user types: + +```shell +$ nix build n +``` + +we need to display a list of flakes starting with `n`. + +### Wrong input + +As we all know we humans make mistakes, all the time. When a typo - intentional +or unintentional - is made, we should prompt for closest possible options or +point to the documentation which would educate user to not make the same +errors. Here are few examples: + +In first example we prompt the user for typing wrong command name: + + +```shell +$ nix int +------------------------------------------------------------------------ + Error! Command `int` not found. +------------------------------------------------------------------------ + Did you mean: + |> nix init + |> nix input +``` + +Sometimes users will make mistake either because of a typo or simply because of +lack of discoverability. Our handling of this cases needs to be context +sensitive. + + +```shell +$ nix init --template=template#pyton +------------------------------------------------------------------------ + Error! Template `template#pyton` not found. +------------------------------------------------------------------------ +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> template#pyton + template#python-pip + template#python-poetry +``` + +### Next steps + +It can be invaluable to newcomers to show what a possible next steps and what +is the usual development workflow with Nix. For example: + + +```shell +$ nix init --template=template#python +Initializing project `template#python` + in `/home/USER/dev/new-project` + + Next steps + |> nix develop -- to enter development environment + |> nix build -- to build your project +``` + +### Educate the user + +We should take any opportunity to **educate users**, but at the same time we +must **be very very careful to not annoy users**. There is a thin line between +being helpful and being annoying. + +An example of educating users might be to provide *Tips* in places where they +are waiting. + +```shell +$ nix build + Started building my-project 1.2.3 + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +Now **Learn** part of the output is where you educate users. You should only +show it when you know that a build will take some time and not annoy users of +the builds that take only few seconds. + +Every feature like this should go though a intensive review and testing to +collect as much a feedback as possible and to fine tune every little detail. If +done right this can be an awesome features beginners and advance users will +love, but if not done perfectly it will annoy users and leave bad impression. + +# Input + +Input to a command is provided via `ARGUMENTS` and `OPTIONS`. + +`ARGUMENTS` represent a required input for a function. When choosing to use +`ARGUMENT` over function please be aware of the downsides that come with it: + +- User will need to remember the order of `ARGUMENTS`. This is not a problem if + there is only one `ARGUMENT`. +- With `OPTIONS` it is possible to provide much better auto completion. +- With `OPTIONS` it is possible to provide much better error message. +- Using `OPTIONS` it will mean there is a little bit more typing. + +We don’t discourage the use of `ARGUMENTS`, but simply want to make every +developer consider the downsides and choose wisely. + +## Naming the `OPTIONS` + +Then only naming convention - apart from the ones mentioned in Naming the +`COMMANDS` section is how flags are named. + +Flags are a type of `OPTION` that represent an option that can be turned ON of +OFF. We can say **flags are boolean type of** `**OPTION**`. + +Here are few examples of flag `OPTIONS`: + +- `--colors` vs. `--no-colors` (showing colors in the output) +- `--emojis` vs. `--no-emojis` (showing emojis in the output) + +## Prompt when input not provided + +For **Tier1** commands we want command to improve the discoverability of +possible input. A new user will most likely not know which `ARGUMENTS` and +`OPTIONS` are required or which values are possible for those options. + +In cases, the user might not provide the input or they provide wrong input, +rather then show the error, prompt a user with an option to find and select +correct input (see examples). + +Prompting is of course not required when TTY is not attached to STDIN. This +would mean that scripts wont need to handle prompt, but rather handle errors. + +A place to use prompt and provide user with interactive select + + +```shell +$ nix init +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> py + template#python-pip + template#python-poetry + [ Showing 2 templates from 1345 templates ] +``` + +Another great place to add prompts are **confirmation dialogues for dangerous +actions**. For example when adding new substitutor via `OPTIONS` or via +`flake.nix` we should prompt - for the first time - and let user review what is +going to happen. + + +```shell +$ nix build --option substitutors https://cache.example.org +------------------------------------------------------------------------ + Warning! A security related question need to be answered. +------------------------------------------------------------------------ + The following substitutors will be used to in `my-project`: + - https://cache.example.org + + Do you allow `my-project` to use above mentioned substitutors? + [y/N] |> y +``` + +# Output + +Terminal output can be quite limiting in many ways. Which should forces us to +think about the experience even more. As with every design the output is a +compromise between being terse and being verbose, between showing help to +beginners and annoying advance users. For this it is important that we know +what are the priorities. + +Nix command line should be first and foremost written with beginners in mind. +But users wont stay beginners for long and what was once useful might quickly +become annoying. There is no golden rule that we can give in this guideline +that would make it easier how to draw a line and find best compromise. + +What we would encourage is to **build prototypes**, do some **user testing** +and collect **feedback**. Then repeat the cycle few times. + +First design the *happy path* and only after your iron it out, continue to work +on **edge cases** (handling and displaying errors, changes of the output by +certain `OPTIONS`, etc…) + +## Follow best practices + +Needless to say we Nix must be a good citizen and follow best practices in +command line. + +In short: **STDOUT is for output, STDERR is for (human) messaging.** + +STDOUT and STDERR provide a way for you to output messages to the user while +also allowing them to redirect content to a file. For example: + +```shell +$ nix build > build.txt +------------------------------------------------------------------------ + Error! Atrribute `bin` missing at (1:94) from string. +------------------------------------------------------------------------ + + 1| with import { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } "" +``` + +Because this warning is on STDERR, it doesn’t end up in the file. + +But not everything on STDERR is an error though. For example, you can run `nix +build` and collect logs in a file while still seeing the progress. + +``` +$ nix build > build.txt + Evaluated 1234 files in 1.2 seconds + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +## Errors (WIP) + +**TODO**: Once we have implementation for the *happy path* then we will think +how to present errors. + +## Not only for humans + +Terse, machine-readable output formats can also be useful but shouldn’t get in +the way of making beautiful CLI output. When needed, commands should offer a +`--json` flag to allow users to easily parse and script the CLI. + +When TTY is not detected on STDOUT we should remove all design elements (no +colors, no emojis and using ASCII instead of Unicode symbols). The same should +happen when TTY is not detected on STDERR. We should not display progress / +status section, but only print warnings and errors. + +## Dialog with the user + +CLIs don't always make it clear when an action has taken place. For every +action a user performs, your CLI should provide an equal and appropriate +reaction, clearly highlighting the what just happened. For example: + +```shell +$ nix build + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +... + Success! You have successfully built my-project. +$ +``` + +Above command clearly states that command successfully completed. And in case +of `nix build`, which is a command that might take some time to complete, it is +equally important to also show that a command started. + +## Text alignment + +Text alignment is the number one design element that will present all of the +Nix commands as a family and not as separate tools glued together. + +The format we should follow is: + +```shell +$ nix COMMAND + VERB_1 NOUN and other words + VERB__1 NOUN and other words + |> Some details +``` + +Few rules that we can extract from above example: + +- Each line should start at least with one space. +- First word should be a VERB and must be aligned to the right. +- Second word should be a NOUN and must be aligned to the left. +- If you can not find a good VERB / NOUN pair, don’t worry make it as + understandable to the user as possible. +- More details of each line can be provided by `|>` character which is serving + as the first word when aligning the text + +Don’t forget you should also test your terminal output with colors and emojis +off (`--no-colors --no-emojis`). + +## Dim / Bright + +After comparing few terminals with different color schemes we would **recommend +to avoid using dimmed text**. The difference from the rest of the text is very +little in many terminal and color scheme combinations. Sometimes the difference +is not even notable, therefore relying on it wouldn’t make much sense. + +**The bright text is much better supported** across terminals and color +schemes. Most of the time the difference is perceived as if the bright text +would be bold. + +## Colors + +Humans are already conditioned by society to attach certain meaning to certain +colors. While the meaning is not universal, a simple collection of colors is +used to represent basic emotions. + +Colors that can be used in output + +- Red = error, danger, stop +- Green = success, good +- Yellow/Orange = proceed with caution, warning, in progress +- Blue/Magenta = stability, calm + +While colors are nice, when command line is used by machines (in automation +scripts) you want to remove the colors. There should be a global `--no-colors` +option that would remove the colors. + +## Special (Unicode) characters + +Most of the terminal have good support for Unicode characters and you should +use them in your output by default. But always have a backup solution that is +implemented only with ASCII characters and will be used when `--ascii` option +is going to be passed in. Please make sure that you test your output also +without Unicode characters + +More they showing all the different Unicode characters it is important to +**establish common set of characters** that we use for certain situations. + +## Emojis + +Emojis help channel emotions even better than text, colors and special +characters. + +We recommend **keeping the set of emojis to a minimum**. This will enable each +emoji to stand out more. + +As not everybody is happy about emojis we should provide an `--no-emojis` +option to disable them. Please make sure that you test your output also without +emojis. + +## Tables + +All commands that are listing certain data can be implemented in some sort of a +table. It’s important that each row of your output is a single ‘entry’ of data. +Never output table borders. It’s noisy and a huge pain for parsing using other +tools such as `grep`. + +Be mindful of the screen width. Only show a few columns by default with the +table header, for more the table can be manipulated by the following options: + +- `--no-headers`: Show column headers by default but allow to hide them. +- `--columns`: Comma-separated list of column names to add. +- `--sort`: Allow sorting by column. Allow inverse and multi-column sort as well. + +## Interactive output + +Interactive output was selected to be able to strike the balance between +beginners and advance users. While the default output will target beginners it +can, with a few key strokes, be changed into and advance introspection tool. + +### Progress + +For longer running commands we should provide and overview of the progress. +This is shown best in `nix build` example: + +```shell +$ nix build + Started building my-project 1.2.3 + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +### Search + +Use a `fzf` like fuzzy search when there are multiple options to choose from. + +```shell +$ nix init +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> py + template#python-pip + template#python-poetry + [ Showing 2 templates from 1345 templates ] +``` + +### Prompt + +In some situations we need to prompt the user and inform the user about what is +going to happen. + +```shell +$ nix build --option substitutors https://cache.example.org +------------------------------------------------------------------------ + Warning! A security related question need to be answered. +------------------------------------------------------------------------ + The following substitutors will be used to in `my-project`: + - https://cache.example.org + + Do you allow `my-project` to use above mentioned substitutors? + [y/N] |> y +``` + +## Verbosity + +There are many ways that you can control verbosity. + +Verbosity levels are: + +- `ERROR` (level 0) +- `WARN` (level 1) +- `NOTICE` (level 2) +- `INFO` (level 3) +- `TALKATIVE` (level 4) +- `CHATTY` (level 5) +- `DEBUG` (level 6) +- `VOMIT` (level 7) + +The default level that the command starts is `ERROR`. The simplest way to +increase the verbosity by stacking `-v` option (eg: `-vvv == level 3 == INFO`). +There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and +`--quiet` to run in `ERROR` verbosity level. + +---------- + +# Appendix 1: Commands naming exceptions + +`nix init` and `nix repl` are well established From 148608ba6ddf93168e86525627bed755a474d21f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 12:26:09 +0100 Subject: [PATCH 175/227] Add 'nix help' --- src/nix/main.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5056ceb78..a75f8ae65 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -149,6 +149,50 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs } }; +static void showHelp(std::vector subcommand) +{ + showManPage(subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand))); +} + +struct CmdHelp : Command +{ + std::vector subcommand; + + CmdHelp() + { + expectArgs({ + .label = "subcommand", + .handler = {&subcommand}, + }); + } + + std::string description() override + { + return "show help about 'nix' or a particular subcommand"; + } + + Examples examples() override + { + return { + Example{ + "To show help about 'nix' in general:", + "nix help" + }, + Example{ + "To show help about a particular subcommand:", + "nix help run" + }, + }; + } + + void run() override + { + showHelp(subcommand); + } +}; + +static auto rCmdHelp = registerCommand("help"); + void mainWrapped(int argc, char * * argv) { /* The chroot helper needs to be run before any threads have been From df552a26452f4cdf734edcac049187b8fd806153 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 21:25:32 +0100 Subject: [PATCH 176/227] nix eval: Add option to write a directory This is useful for generating the nix manpages, but it may have other applications (like generating configuration files without a Nix store). --- src/nix/eval.cc | 55 ++++++++++++++++++++++++++++++++++++++++++---- tests/pure-eval.sh | 8 +++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 8da81d667..0f02919de 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -3,6 +3,7 @@ #include "shared.hh" #include "store-api.hh" #include "eval.hh" +#include "eval-inline.hh" #include "json.hh" #include "value-to-json.hh" #include "progress-bar.hh" @@ -13,6 +14,7 @@ struct CmdEval : MixJSON, InstallableCommand { bool raw = false; std::optional apply; + std::optional writeTo; CmdEval() { @@ -24,6 +26,13 @@ struct CmdEval : MixJSON, InstallableCommand .labels = {"expr"}, .handler = {&apply}, }); + + addFlag({ + .longName = "write-to", + .description = "write a string or attrset of strings to 'path'", + .labels = {"path"}, + .handler = {&writeTo}, + }); } std::string description() override @@ -66,7 +75,7 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state).first; + auto [v, pos] = installable->toValue(*state); PathSet context; if (apply) { @@ -77,13 +86,51 @@ struct CmdEval : MixJSON, InstallableCommand v = vRes; } - if (raw) { + if (writeTo) { + stopProgressBar(); + + if (pathExists(*writeTo)) + throw Error("path '%s' already exists", *writeTo); + + std::function recurse; + + recurse = [&](Value & v, const Pos & pos, const Path & path) + { + state->forceValue(v); + if (v.type == tString) + // FIXME: disallow strings with contexts? + writeFile(path, v.string.s); + else if (v.type == tAttrs) { + if (mkdir(path.c_str(), 0777) == -1) + throw SysError("creating directory '%s'", path); + for (auto & attr : *v.attrs) + try { + if (attr.name == "." || attr.name == "..") + throw Error("invalid file name '%s'", attr.name); + recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name)); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name)); + throw; + } + } + else + throw TypeError("value at '%s' is not a string or an attribute set", pos); + }; + + recurse(*v, pos, *writeTo); + } + + else if (raw) { stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); - } else if (json) { + } + + else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); - } else { + } + + else { state->forceValueDeep(*v); logger->cout("%s", *v); } diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 43a765997..561ca53fb 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -16,3 +16,11 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x" + +rm -rf $TEST_ROOT/eval-out +nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' +[[ $(cat $TEST_ROOT/eval-out/x) = foobar ]] +[[ $(cat $TEST_ROOT/eval-out/y/z) = bla ]] + +rm -rf $TEST_ROOT/eval-out +(! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') From 72428e38d904a1db746f37c8727ab7b0fbb457bc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 23:05:28 +0100 Subject: [PATCH 177/227] Generate separate manpages for each nix subcommand --- .gitignore | 3 +- doc/manual/generate-manpage.nix | 50 +++++++++++++------- doc/manual/local.mk | 28 ++++++++--- doc/manual/src/{SUMMARY.md => SUMMARY.md.in} | 2 +- 4 files changed, 58 insertions(+), 25 deletions(-) rename doc/manual/src/{SUMMARY.md => SUMMARY.md.in} (99%) diff --git a/.gitignore b/.gitignore index c51582cf0..37aada307 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json -/doc/manual/src/command-ref/nix.md +/doc/manual/src/SUMMARY.md +/doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/expressions/builtins.md diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index db266750a..2801e96fa 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,34 +1,40 @@ +command: + with builtins; with import ./utils.nix; let showCommand = - { command, section, def }: - "${section} Name\n\n" + { command, def, filename }: + "# Name\n\n" + "`${command}` - ${def.description}\n\n" - + "${section} Synopsis\n\n" + + "# Synopsis\n\n" + showSynopsis { inherit command; args = def.args; } + + (if def.commands or {} != {} + then + "where *subcommand* is one of the following:\n\n" + # FIXME: group by category + + concatStrings (map (name: + "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n") + (attrNames def.commands)) + + "\n" + else "") + (if def ? doc - then "${section} Description\n\n" + def.doc + "\n\n" + then "# Description\n\n" + def.doc + "\n\n" else "") + (let s = showFlags def.flags; in if s != "" - then "${section} Flags\n\n${s}" + then "# Flags\n\n${s}" else "") + (if def.examples or [] != [] then - "${section} Examples\n\n" + "# Examples\n\n" + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else "") - + (if def.commands or [] != [] - then concatStrings ( - map (name: - "# Subcommand `${command} ${name}`\n\n" - + showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; }) - (attrNames def.commands)) else ""); + appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; + showFlags = flags: concatStrings (map (longName: @@ -48,8 +54,20 @@ let "`${command}` [*flags*...] ${concatStringsSep " " (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; + processCommand = { command, def, filename }: + [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ] + ++ concatMap + (name: processCommand { + filename = appendName filename name; + command = command + " " + name; + def = def.commands.${name}; + }) + (attrNames def.commands or {}); + in -command: - -showCommand { command = "nix"; section = "#"; def = command; } +let + manpages = processCommand { filename = "nix"; command = "nix"; def = command; }; + summary = concatStrings (map (manpage: " - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages); +in +(listToAttrs manpages) // { "SUMMARY.md" = summary; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index bb8b3b60a..b40fa4ed2 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -4,7 +4,7 @@ MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md) # Generate man pages. man-pages := $(foreach n, \ - nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 nix.1 \ + nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ nix-collect-garbage.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-hash.1 nix-copy-closure.1 \ @@ -22,7 +22,7 @@ dummy-env = env -i \ NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ NIX_STATE_DIR=/dummy -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @@ -42,13 +42,17 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(trace-gen) lowdown -sT man $^.tmp -o $@ @rm $^.tmp -$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix - $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli + $(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp @mv $@.tmp $@ +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix + @rm -rf $@ + $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' + $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -61,7 +65,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -71,7 +75,17 @@ $(d)/builtins.json: $(bindir)/nix # Generate the HTML manual. install: $(docdir)/manual/index.html -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/command-ref/nix.md $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md +# Generate 'nix' manpages. +install: $(d)/src/command-ref/new-cli + for i in doc/manual/src/command-ref/new-cli/*.md; do \ + name=$$(basename $$i .md); \ + if [[ $$name = SUMMARY ]]; then continue; fi; \ + printf "Title: %s\n\n" "$$name" > $$i.tmp; \ + cat $$i >> $$i.tmp; \ + lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \ + done + +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(trace-gen) mdbook build doc/manual -d $(docdir)/manual @cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md.in similarity index 99% rename from doc/manual/src/SUMMARY.md rename to doc/manual/src/SUMMARY.md.in index 8281f683f..b5ae34cfa 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md.in @@ -62,7 +62,7 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) - - [nix](command-ref/nix.md) +@manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) From e2efc63979a5485a96accaa1e49ffec65c353078 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 23:19:16 +0100 Subject: [PATCH 178/227] Put examples first in the manpages --- doc/manual/generate-manpage.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 2801e96fa..4a0b0290b 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -20,6 +20,11 @@ let (attrNames def.commands)) + "\n" else "") + + (if def.examples or [] != [] + then + "# Examples\n\n" + + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) + else "") + (if def ? doc then "# Description\n\n" + def.doc + "\n\n" else "") @@ -27,11 +32,7 @@ let if s != "" then "# Flags\n\n${s}" else "") - + (if def.examples or [] != [] - then - "# Examples\n\n" - + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else ""); + ; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; From b0de7b20160ba1910987c9165bc88689e6587c41 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 17:16:50 -0600 Subject: [PATCH 179/227] Check for rosetta 2 support before installing --- scripts/install.in | 5 +++++ 1 file changed, 5 insertions(+) mode change 100644 => 100755 scripts/install.in diff --git a/scripts/install.in b/scripts/install.in old mode 100644 new mode 100755 index 7c3b795cc..46d3817b9 --- a/scripts/install.in +++ b/scripts/install.in @@ -46,6 +46,11 @@ case "$(uname -s).$(uname -m)" in system=x86_64-darwin ;; Darwin.arm64) + # check for Rosetta 2 support + if ! [ -d /Library/Apple/usr/libexec/oah ]; then + oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" + fi + hash=@binaryTarball_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ # eventually maybe: arm64-darwin From addf9f4edeab8e8e2cb115dd98dc764cc1f19fcf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 19:05:02 -0600 Subject: [PATCH 180/227] Call it aarch64-darwin instead of arm64-darwin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gnu-config standardized on aarch64 for machine name so host_cpu part of $system will always be aarch64. That means system will be aarch64-darwin too. uname however could report either “aarch64” (if gnu coreutils) or “arm64” (if apple’s uname). We should support both for compatiblity here. --- scripts/install.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 46d3817b9..9c8831d2b 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -45,7 +45,7 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_x86_64-darwin@ system=x86_64-darwin ;; - Darwin.arm64) + Darwin.arm64|Darwin.aarch64) # check for Rosetta 2 support if ! [ -d /Library/Apple/usr/libexec/oah ]; then oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" @@ -53,7 +53,7 @@ case "$(uname -s).$(uname -m)" in hash=@binaryTarball_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ - # eventually maybe: arm64-darwin + # eventually maybe: aarch64-darwin system=x86_64-darwin ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; From 94f359525ed913635eb21e8599d880db877bcc2a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 19:14:34 -0600 Subject: [PATCH 181/227] Update config.guess for proper arm64 macOS detection This fixes results for arm64 macOS so config.guess now reports: aarch64-apple-darwin20.1.0 instead of arm-apple-darwin20.1.0 --- config/config.guess | 538 +++++++++---- config/config.sub | 1756 ++++++++++++++++++++++--------------------- 2 files changed, 1271 insertions(+), 1023 deletions(-) diff --git a/config/config.guess b/config/config.guess index d4fb3213e..699b3a10b 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-08-02' +timestamp='2020-11-19' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -27,12 +27,12 @@ timestamp='2018-08-02' # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess # # Please send patches to . -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ Usage: $0 [OPTION] @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -96,13 +96,14 @@ fi tmp= # shellcheck disable=SC2172 -trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15 -trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0 +trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 : "${TMPDIR=/tmp}" # shellcheck disable=SC2039 - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } @@ -130,16 +131,14 @@ if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi -UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown -UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown -UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown +UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown +UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown +UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - LIBC=gnu + LIBC=unknown set_cc_for_build cat <<-EOF > "$dummy.c" @@ -148,17 +147,29 @@ Linux|GNU|GNU/*) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc - #else + #elif defined(__GLIBC__) LIBC=gnu + #else + #include + /* First heuristic to detect musl libc. */ + #ifdef __DEFINED_va_list + LIBC=musl + #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')" - # If ldd exists, use it to detect musl libc. - if command -v ldd >/dev/null && \ - ldd --version 2>&1 | grep -q ^musl - then - LIBC=musl + # Second heuristic to detect musl libc. + if [ "$LIBC" = unknown ] && + command -v ldd >/dev/null && + ldd --version 2>&1 | grep -q ^musl; then + LIBC=musl + fi + + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + if [ "$LIBC" = unknown ]; then + LIBC=gnu fi ;; esac @@ -178,19 +189,20 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ "/sbin/$sysctl" 2>/dev/null || \ "/usr/sbin/$sysctl" 2>/dev/null || \ - echo unknown)` + echo unknown)) case "$UNAME_MACHINE_ARCH" in + aarch64eb) machine=aarch64_be-unknown ;; armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) - arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` - endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,') + endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p') machine="${arch}${endian}"-unknown ;; *) machine="$UNAME_MACHINE_ARCH"-unknown ;; @@ -221,7 +233,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in case "$UNAME_MACHINE_ARCH" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr") ;; esac # The OS release @@ -234,7 +246,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in release='-gnu' ;; *) - release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` + release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2) ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: @@ -243,15 +255,15 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//') echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//') echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" exit ;; *:LibertyBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//') echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" exit ;; *:MidnightBSD:*:*) @@ -263,6 +275,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:SolidBSD:*:*) echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; + *:OS108:*:*) + echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" + exit ;; macppc:MirBSD:*:*) echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; @@ -272,26 +287,29 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:Sortix:*:*) echo "$UNAME_MACHINE"-unknown-sortix exit ;; + *:Twizzler:*:*) + echo "$UNAME_MACHINE"-unknown-twizzler + exit ;; *:Redox:*:*) echo "$UNAME_MACHINE"-unknown-redox exit ;; mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}') ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}') ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on # OSF/1 and Tru64 systems produced since 1995. I hope that # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1) case "$ALPHA_CPU_TYPE" in "EV4 (21064)") UNAME_MACHINE=alpha ;; @@ -329,7 +347,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" + echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -363,7 +381,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "`(/bin/universe) 2>/dev/null`" = att ; then + if test "$( (/bin/universe) 2>/dev/null)" = att ; then echo pyramid-pyramid-sysv3 else echo pyramid-pyramid-bsd @@ -376,54 +394,59 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo sparc-icl-nx6 exit ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case `/usr/bin/uname -p` in + case $(/usr/bin/uname -p) in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" - case `isainfo -b` in - 32) - echo i386-pc-solaris2"$UNAME_REL" - ;; - 64) - echo x86_64-pc-solaris2"$UNAME_REL" - ;; - esac + set_cc_for_build + SUN_ARCH=i386 + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH=x86_64 + fi + fi + echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:*:*) - case "`/usr/bin/arch -k`" in + case "$(/usr/bin/arch -k)" in Series*|S4*) - UNAME_RELEASE=`uname -v` + UNAME_RELEASE=$(uname -v) ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" + echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')" exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) - UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null) test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case "`/bin/arch`" in + case "$(/bin/arch)" in sun3) echo m68k-sun-sunos"$UNAME_RELEASE" ;; @@ -503,8 +526,8 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in } EOF $CC_FOR_BUILD -o "$dummy" "$dummy.c" && - dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`"$dummy" "$dummyarg"` && + dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') && + SYSTEM_NAME=$("$dummy" "$dummyarg") && { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos"$UNAME_RELEASE" exit ;; @@ -531,11 +554,11 @@ EOF exit ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] + UNAME_PROCESSOR=$(/usr/bin/uname -p) + if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 then - if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ - [ "$TARGET_BINARY_INTERFACE"x = x ] + if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ + test "$TARGET_BINARY_INTERFACE"x = x then echo m88k-dg-dgux"$UNAME_RELEASE" else @@ -559,17 +582,17 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" + echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix exit ;; ia64:AIX:*:*) - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if test -x /usr/bin/oslevel ; then + IBM_REV=$(/usr/bin/oslevel) else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi @@ -589,7 +612,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") then echo "$SYSTEM_NAME" else @@ -602,15 +625,15 @@ EOF fi exit ;; *:AIX:*:[4567]) - IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }') if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/lslpp ] ; then - IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` + if test -x /usr/bin/lslpp ; then + IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/) else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi @@ -638,14 +661,14 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') case "$UNAME_MACHINE" in 9000/31?) HP_ARCH=m68000 ;; 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) - if [ -x /usr/bin/getconf ]; then - sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + if test -x /usr/bin/getconf; then + sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null) + sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null) case "$sc_cpu_version" in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 @@ -657,7 +680,7 @@ EOF esac ;; esac fi - if [ "$HP_ARCH" = "" ]; then + if test "$HP_ARCH" = ""; then set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" @@ -692,11 +715,11 @@ EOF exit (0); } EOF - (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy") test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ "$HP_ARCH" = hppa2.0w ] + if test "$HP_ARCH" = hppa2.0w then set_cc_for_build @@ -720,7 +743,7 @@ EOF echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) @@ -750,7 +773,7 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; @@ -770,7 +793,7 @@ EOF echo hppa1.0-hp-osf exit ;; i*86:OSF1:*:*) - if [ -x /usr/sbin/sysversion ] ; then + if test -x /usr/sbin/sysversion ; then echo "$UNAME_MACHINE"-unknown-osf1mk else echo "$UNAME_MACHINE"-unknown-osf1 @@ -819,14 +842,14 @@ EOF echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` + FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz) + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/') echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/') echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -838,26 +861,26 @@ EOF *:BSD/OS:*:*) echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" exit ;; - arm*:FreeBSD:*:*) - UNAME_PROCESSOR=`uname -p` + arm:FreeBSD:*:*) + UNAME_PROCESSOR=$(uname -p) set_cc_for_build if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi else - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf fi exit ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=$(/usr/bin/uname -p) case "$UNAME_PROCESSOR" in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac - echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; i*:CYGWIN*:*) echo "$UNAME_MACHINE"-pc-cygwin @@ -890,18 +913,18 @@ EOF echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-unknown-cygwin + echo x86_64-pc-cygwin exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; *:GNU:*:*) # the GNU system - echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" + echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" + echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC" exit ;; *:Minix:*:*) echo "$UNAME_MACHINE"-unknown-minix @@ -914,7 +937,7 @@ EOF echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -981,22 +1004,50 @@ EOF exit ;; mips:Linux:*:* | mips64:Linux:*:*) set_cc_for_build + IS_GLIBC=0 + test x"${LIBC}" = xgnu && IS_GLIBC=1 sed 's/^ //' << EOF > "$dummy.c" #undef CPU - #undef ${UNAME_MACHINE} - #undef ${UNAME_MACHINE}el + #undef mips + #undef mipsel + #undef mips64 + #undef mips64el + #if ${IS_GLIBC} && defined(_ABI64) + LIBCABI=gnuabi64 + #else + #if ${IS_GLIBC} && defined(_ABIN32) + LIBCABI=gnuabin32 + #else + LIBCABI=${LIBC} + #endif + #endif + + #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa64r6 + #else + #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa32r6 + #else + #if defined(__mips64) + CPU=mips64 + #else + CPU=mips + #endif + #endif + #endif + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=${UNAME_MACHINE}el + MIPS_ENDIAN=el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=${UNAME_MACHINE} + MIPS_ENDIAN= #else - CPU= + MIPS_ENDIAN= #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" - test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')" + test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; mips64el:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -1015,7 +1066,7 @@ EOF exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level - case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; *) echo hppa-unknown-linux-"$LIBC" ;; @@ -1055,7 +1106,17 @@ EOF echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" + set_cc_for_build + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_X32 >/dev/null + then + LIBCABI="$LIBC"x32 + fi + fi + echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" exit ;; xtensa*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -1095,7 +1156,7 @@ EOF echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; i*86:*:4.*:*) - UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` + UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//') if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else @@ -1104,19 +1165,19 @@ EOF exit ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. - case `/bin/uname -X | grep "^Machine"` in + case $(/bin/uname -X | grep "^Machine") in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then - UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //')) (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ && UNAME_MACHINE=i586 @@ -1166,7 +1227,7 @@ EOF 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ @@ -1177,7 +1238,7 @@ EOF NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ @@ -1210,7 +1271,7 @@ EOF exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=`(uname -p) 2>/dev/null` + UNAME_MACHINE=$( (uname -p) 2>/dev/null) echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv @@ -1244,7 +1305,7 @@ EOF echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if [ -d /usr/nec ]; then + if test -d /usr/nec; then echo mips-nec-sysv"$UNAME_RELEASE" else echo mips-unknown-sysv"$UNAME_RELEASE" @@ -1292,44 +1353,48 @@ EOF *:Rhapsody:*:*) echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; + arm64:Darwin:*:*) + echo aarch64-apple-darwin"$UNAME_RELEASE" + exit ;; *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - set_cc_for_build - if test "$UNAME_PROCESSOR" = unknown ; then - UNAME_PROCESSOR=powerpc + UNAME_PROCESSOR=$(uname -p) + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + if command -v xcode-select > /dev/null 2> /dev/null && \ + ! xcode-select --print-path > /dev/null 2> /dev/null ; then + # Avoid executing cc if there is no toolchain installed as + # cc will be a stub that puts up a graphical alert + # prompting the user to install developer tools. + CC_FOR_BUILD=no_compiler_found + else + set_cc_for_build fi - if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then - if [ "$CC_FOR_BUILD" != no_compiler_found ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc fi elif test "$UNAME_PROCESSOR" = i386 ; then - # Avoid executing cc on OS X 10.9, as it ships with a stub - # that puts up a graphical alert prompting to install - # developer tools. Any system running Mac OS X 10.7 or - # later (Darwin 11 and later) is required to have a 64-bit - # processor. This is not true of the ARM version of Darwin - # that Apple uses in portable devices. - UNAME_PROCESSOR=x86_64 + # uname -m returns i386 or x86_64 + UNAME_PROCESSOR=$UNAME_MACHINE fi echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=`uname -p` + UNAME_PROCESSOR=$(uname -p) if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc @@ -1397,10 +1462,10 @@ EOF echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; *:*VMS:*:*) - UNAME_MACHINE=`(uname -p) 2>/dev/null` + UNAME_MACHINE=$( (uname -p) 2>/dev/null) case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; @@ -1410,7 +1475,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" + echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')" exit ;; i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos @@ -1424,8 +1489,148 @@ EOF amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs exit ;; + *:Unleashed:*:*) + echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" + exit ;; esac +# No uname command or uname output not recognized. +set_cc_for_build +cat > "$dummy.c" < +#include +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#include +#if defined(_SIZE_T_) || defined(SIGLOST) +#include +#endif +#endif +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null); + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); +#endif + +#if defined (vax) +#if !defined (ultrix) +#include +#if defined (BSD) +#if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +#else +#if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#endif +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#else +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname un; + uname (&un); + printf ("vax-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("vax-dec-ultrix\n"); exit (0); +#endif +#endif +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname *un; + uname (&un); + printf ("mips-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("mips-dec-ultrix\n"); exit (0); +#endif +#endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. +test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } + echo "$0: unable to guess system type" >&2 case "$UNAME_MACHINE:$UNAME_SYSTEM" in @@ -1445,9 +1650,15 @@ This script (version $timestamp), has failed to recognize the operating system you are using. If your script is old, overwrite *all* copies of config.guess and config.sub with the latest versions from: - https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess + https://git.savannah.gnu.org/cgit/config.git/plain/config.guess and - https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub + https://git.savannah.gnu.org/cgit/config.git/plain/config.sub +EOF + +year=$(echo $timestamp | sed 's,-.*,,') +# shellcheck disable=SC2003 +if test "$(expr "$(date +%Y)" - "$year")" -lt 3 ; then + cat >&2 </dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` +uname -m = $( (uname -m) 2>/dev/null || echo unknown) +uname -r = $( (uname -r) 2>/dev/null || echo unknown) +uname -s = $( (uname -s) 2>/dev/null || echo unknown) +uname -v = $( (uname -v) 2>/dev/null || echo unknown) -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null` +/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null) +/bin/uname -X = $( (/bin/uname -X) 2>/dev/null) -hostinfo = `(hostinfo) 2>/dev/null` -/bin/universe = `(/bin/universe) 2>/dev/null` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` -/bin/arch = `(/bin/arch) 2>/dev/null` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` +hostinfo = $( (hostinfo) 2>/dev/null) +/bin/universe = $( (/bin/universe) 2>/dev/null) +/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null) +/bin/arch = $( (/bin/arch) 2>/dev/null) +/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null) +/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null) UNAME_MACHINE = "$UNAME_MACHINE" UNAME_RELEASE = "$UNAME_RELEASE" UNAME_SYSTEM = "$UNAME_SYSTEM" UNAME_VERSION = "$UNAME_VERSION" EOF +fi exit 1 diff --git a/config/config.sub b/config/config.sub index c19e67180..19c9553b1 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-08-13' +timestamp='2020-12-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -33,7 +33,7 @@ timestamp='2018-08-13' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub +# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -50,7 +50,7 @@ timestamp='2018-08-13' # CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM # It is wrong to echo any other type of specification. -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS @@ -67,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -89,7 +89,7 @@ while test $# -gt 0 ; do - ) # Use stdin as input. break ;; -* ) - echo "$me: invalid option $1$help" + echo "$me: invalid option $1$help" >&2 exit 1 ;; *local*) @@ -111,7 +111,8 @@ case $# in esac # Split fields of configuration type -IFS="-" read -r field1 field2 field3 field4 <&2 - exit 1 + # Recognize the canonical CPU types that are allowed with any + # company name. + case $cpu in + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ + | alphapca5[67] | alpha64pca5[67] \ + | am33_2.0 \ + | amdgcn \ + | arc | arceb \ + | arm | arm[lb]e | arme[lb] | armv* \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin | bpf | bs2000 \ + | c[123]* | c30 | [cjt]90 | c4x \ + | c8051 | clipper | craynv | csky | cydra \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | elxsi | epiphany \ + | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | h8300 | h8500 \ + | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i*86 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle \ + | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ + | m88110 | m88k | maxq | mb | mcore | mep | metag \ + | microblaze | microblazeel \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64eb | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mmix \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | none | np1 | ns16k | ns32k | nvptx \ + | open8 \ + | or1k* \ + | or32 \ + | orion \ + | picochip \ + | pdp10 | pdp11 | pj | pjl | pn | power \ + | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | romp | rs6000 | rx \ + | s390 | s390x \ + | score \ + | sh | shl \ + | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ + | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | spu \ + | tahoe \ + | thumbv7* \ + | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tron \ + | ubicom32 \ + | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | vax \ + | visium \ + | w65 \ + | wasm32 | wasm64 \ + | we32k \ + | x86 | x86_64 | xc16x | xgate | xps100 \ + | xstormy16 | xtensa* \ + | ymp \ + | z8k | z80) + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + exit 1 + ;; + esac ;; esac # Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` +case $vendor in + digital*) + vendor=dec ;; - *-commodore*) - basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` + commodore*) + vendor=cbm ;; *) ;; @@ -1296,8 +1279,47 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x$os != x ] +if test x$basic_os != x then + +# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# set os. +case $basic_os in + gnu/linux*) + kernel=linux + os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|') + ;; + os2-emx) + kernel=os2 + os=$(echo $basic_os | sed -e 's|os2-emx|emx|') + ;; + nto-qnx*) + kernel=nto + os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|') + ;; + *-*) + # shellcheck disable=SC2162 + IFS="-" read kernel os <&2 - exit 1 + # No normalization, but not necessarily accepted, that comes below. ;; esac + else # Here we handle the default operating systems that come with various machines. @@ -1551,7 +1498,8 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. -case $basic_machine in +kernel= +case $cpu-$vendor in score-*) os=elf ;; @@ -1562,7 +1510,8 @@ case $basic_machine in os=riscix1.2 ;; arm*-rebel) - os=linux + kernel=linux + os=gnu ;; arm*-semi) os=aout @@ -1728,86 +1677,173 @@ case $basic_machine in os=none ;; esac + fi -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) - case $os in - riscix*) - vendor=acorn - ;; - sunos*) - vendor=sun - ;; - cnk*|-aix*) - vendor=ibm - ;; - beos*) - vendor=be - ;; - hpux*) - vendor=hp - ;; - mpeix*) - vendor=hp - ;; - hiux*) - vendor=hitachi - ;; - unos*) - vendor=crds - ;; - dgux*) - vendor=dg - ;; - luna*) - vendor=omron - ;; - genix*) - vendor=ns - ;; - clix*) - vendor=intergraph - ;; - mvs* | opened*) - vendor=ibm - ;; - os400*) - vendor=ibm - ;; - ptx*) - vendor=sequent - ;; - tpf*) - vendor=ibm - ;; - vxsim* | vxworks* | windiss*) - vendor=wrs - ;; - aux*) - vendor=apple - ;; - hms*) - vendor=hitachi - ;; - mpw* | macos*) - vendor=apple - ;; - *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) - vendor=atari - ;; - vos*) - vendor=stratus - ;; - esac - basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` +# Now, validate our (potentially fixed-up) OS. +case $os in + # Sometimes we do "kernel-abi", so those need to count as OSes. + musl* | newlib* | uclibc*) + ;; + # Likewise for "kernel-libc" + eabi | eabihf | gnueabi | gnueabihf) + ;; + # Now accept the basic system types. + # The portable systems comes first. + # Each alternative MUST end in a * to match a version number. + gnu* | android* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]* \ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ + | hiux* | abug | nacl* | netware* | windows* \ + | os9* | macos* | osx* | ios* \ + | mpw* | magic* | mmixware* | mon960* | lnews* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* | twizzler* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | mirbsd* | netbsd* | dicos* | openedition* | ose* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \ + | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | mint* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* | morphos* \ + | scout* | superux* | sysv* | rtmk* | tpf* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx*) + ;; + # This one is extra strict with allowed versions + sco3.2v2 | sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + none) + ;; + *) + echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + exit 1 ;; esac -echo "$basic_machine-$os" +# As a final step for OS-related things, validate the OS-kernel combination +# (given a valid OS), if there is a kernel. +case $kernel-$os in + linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) + ;; + uclinux-uclibc* ) + ;; + -dietlibc* | -newlib* | -musl* | -uclibc* ) + # These are just libc implementations, not actual OSes, and thus + # require a kernel. + echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + exit 1 + ;; + kfreebsd*-gnu* | kopensolaris*-gnu*) + ;; + nto-qnx*) + ;; + os2-emx) + ;; + *-eabi* | *-gnueabi*) + ;; + -*) + # Blank kernel with real OS is always fine. + ;; + *-*) + echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + exit 1 + ;; +esac + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +case $vendor in + unknown) + case $cpu-$os in + *-riscix*) + vendor=acorn + ;; + *-sunos*) + vendor=sun + ;; + *-cnk* | *-aix*) + vendor=ibm + ;; + *-beos*) + vendor=be + ;; + *-hpux*) + vendor=hp + ;; + *-mpeix*) + vendor=hp + ;; + *-hiux*) + vendor=hitachi + ;; + *-unos*) + vendor=crds + ;; + *-dgux*) + vendor=dg + ;; + *-luna*) + vendor=omron + ;; + *-genix*) + vendor=ns + ;; + *-clix*) + vendor=intergraph + ;; + *-mvs* | *-opened*) + vendor=ibm + ;; + *-os400*) + vendor=ibm + ;; + s390-* | s390x-*) + vendor=ibm + ;; + *-ptx*) + vendor=sequent + ;; + *-tpf*) + vendor=ibm + ;; + *-vxsim* | *-vxworks* | *-windiss*) + vendor=wrs + ;; + *-aux*) + vendor=apple + ;; + *-hms*) + vendor=hitachi + ;; + *-mpw* | *-macos*) + vendor=apple + ;; + *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) + vendor=atari + ;; + *-vos*) + vendor=stratus + ;; + esac + ;; +esac + +echo "$cpu-$vendor-${kernel:+$kernel-}$os" exit # Local variables: From 0afab668fa3f20f091dec0a31fc0b0fbaac2afde Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 3 Dec 2020 13:19:20 +0100 Subject: [PATCH 182/227] Don't fail early when -j0 is passed If the build closure contains some CA derivations, then we can't know ahead-of-time that we won't build anything as early-cutoff might come-in at a laster stage --- src/libstore/build/worker.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 1f8999a4b..6c96a93bd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -223,11 +223,6 @@ void Worker::run(const Goals & _topGoals) uint64_t downloadSize, narSize; store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); - debug("entered goal loop"); while (1) { From 8ad72b1f1c272d01d5d67b21553fc261c8df5302 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 3 Dec 2020 13:20:53 +0100 Subject: [PATCH 183/227] Properly test early cutoff with CA derivations Build things with a different seed each time to make sure that it works despite the different drvs --- tests/content-addressed.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 52f7529b5..03eff549c 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -7,30 +7,32 @@ nix --experimental-features 'nix-command ca-derivations' show-derivation --deriv buildAttr () { local derivationPath=$1 - shift - local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + local seedValue=$2 + shift; shift + local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") args+=("$@") nix-build "${args[@]}" } testRemoteCache () { clearCache - local outPath=$(buildAttr dependentNonCA) + local outPath=$(buildAttr dependentNonCA 1) nix copy --to file://$cacheDir $outPath clearStore - buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") + buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") } testDeterministicCA () { - [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]] + [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } testCutoffFor () { local out1 out2 - out1=$(buildAttr $1) + out1=$(buildAttr $1 1) # The seed only changes the root derivation, and not it's output, so the # dependent derivations should only need to be built once. - out2=$(buildAttr $1 -j0) + buildAttr rootCA 2 + out2=$(buildAttr $1 2 -j0) test "$out1" == "$out2" } From c3c858ac6d0c75bd95bd6913276ef20cf2495e96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 13:55:19 +0100 Subject: [PATCH 184/227] Make doc() return arbitrary Markdown rather than the contents of the "Description" section Thus we can return the examples section (and any other sections) from doc() and don't need examples() anymore. --- doc/manual/generate-manpage.nix | 2 +- src/nix/add-to-store.cc | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 4a0b0290b..fbd7f3e7d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -26,7 +26,7 @@ let + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) else "") + (if def ? doc - then "# Description\n\n" + def.doc + "\n\n" + then def.doc + "\n\n" else "") + (let s = showFlags def.flags; in if s != "" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index df51e72d5..022f0ea85 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -39,15 +39,29 @@ struct CmdAddToStore : MixDryRun, StoreCommand std::string doc() override { return R"( + # Description + Copy the file or directory *path* to the Nix store, and print the resulting store path on standard output. - )"; - } - Examples examples() override - { - return { - }; + > **Warning** + > + > The resulting store path is not registered as a garbage + > collector root, so it could be deleted before you have a + > chance to register it. + + # Examples + + Add a regular file to the store: + + ```console + # echo foo > bar + # nix add-to-store --flat ./bar + /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + foo + ``` + )"; } Category category() override { return catUtility; } From 1b0ca3866bae7185e628fda956634c52a5da7a15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 14:23:18 +0100 Subject: [PATCH 185/227] nix add-to-store: Move markdown docs into a separate file --- src/nix/add-to-store.cc | 27 +++------------------------ src/nix/add-to-store.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 src/nix/add-to-store.md diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 022f0ea85..a2721431e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -38,30 +38,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand std::string doc() override { - return R"( - # Description - - Copy the file or directory *path* to the Nix store, and - print the resulting store path on standard output. - - > **Warning** - > - > The resulting store path is not registered as a garbage - > collector root, so it could be deleted before you have a - > chance to register it. - - # Examples - - Add a regular file to the store: - - ```console - # echo foo > bar - # nix add-to-store --flat ./bar - /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - foo - ``` - )"; + return + #include "add-to-store.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/add-to-store.md b/src/nix/add-to-store.md new file mode 100644 index 000000000..593ad67ad --- /dev/null +++ b/src/nix/add-to-store.md @@ -0,0 +1,28 @@ +R""( + +# Description + +Copy the file or directory *path* to the Nix store, and +print the resulting store path on standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a regular file to the store: + +```console +# echo foo > bar + +# nix add-to-store --flat ./bar +/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + +# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar +foo +``` + +)"" From 8ad2c9c4b97f291982598e34530122612c580b83 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 14:35:42 +0100 Subject: [PATCH 186/227] Remove 'dist' target We're not producing source tarballs anymore so this has been bitrotting. --- doc/manual/local.mk | 2 -- local.mk | 6 ------ mk/dist.mk | 17 ----------------- mk/lib.mk | 4 ---- mk/libraries.mk | 1 - mk/programs.mk | 1 - nix-rust/local.mk | 2 -- src/libexpr/local.mk | 2 -- 8 files changed, 35 deletions(-) delete mode 100644 mk/dist.mk diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b40fa4ed2..ee5b328de 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -13,8 +13,6 @@ man-pages := $(foreach n, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 -dist-files += $(man-pages) - # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. dummy-env = env -i \ HOME=/dummy \ diff --git a/local.mk b/local.mk index fe314b902..6a7074e8e 100644 --- a/local.mk +++ b/local.mk @@ -1,9 +1,3 @@ -ifeq ($(MAKECMDGOALS), dist) - dist-files += $(shell cat .dist-files) -endif - -dist-files += configure config.h.in perl/configure - clean-files += Makefile.config GLOBAL_CXXFLAGS += -Wno-deprecated-declarations diff --git a/mk/dist.mk b/mk/dist.mk deleted file mode 100644 index 794b27771..000000000 --- a/mk/dist.mk +++ /dev/null @@ -1,17 +0,0 @@ -ifdef PACKAGE_NAME - -dist-name = $(PACKAGE_NAME)-$(PACKAGE_VERSION) - -dist: $(dist-name).tar.bz2 $(dist-name).tar.xz - -$(dist-name).tar.bz2: $(dist-files) - $(trace-gen) tar cfj $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' - -$(dist-name).tar.xz: $(dist-files) - $(trace-gen) tar cfJ $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' - -clean-files += $(dist-name).tar.bz2 $(dist-name).tar.xz - -print-top-help += echo " dist: Generate a source distribution"; - -endif diff --git a/mk/lib.mk b/mk/lib.mk index 1da51d879..a09ebaa97 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,7 +10,6 @@ bin-scripts := noinst-scripts := man-pages := install-tests := -dist-files := OS = $(shell uname -s) @@ -112,9 +111,6 @@ $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) -include mk/dist.mk - - .PHONY: default all man help all: $(programs-list) $(libs-list) $(jars-list) $(man-pages) diff --git a/mk/libraries.mk b/mk/libraries.mk index e6ef2e3ec..7c0e4f100 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -159,5 +159,4 @@ define build-library libs-list += $$($(1)_PATH) endif clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) - dist-files += $$(_srcs) endef diff --git a/mk/programs.mk b/mk/programs.mk index 3fa9685c3..d0cf5baf0 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -79,7 +79,6 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) - dist-files += $$(_srcs) # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN diff --git a/nix-rust/local.mk b/nix-rust/local.mk index e4bfde31b..50db4783c 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -30,8 +30,6 @@ ifeq ($(OS), Darwin) install_name_tool -id $@ $@ endif -dist-files += $(d)/vendor - clean: clean-rust clean-rust: diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a5422169d..519da33f7 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -35,8 +35,6 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh - $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ From b2d6c6161e61c47f4d8a3faa5799160d100f4e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 18:44:43 +0200 Subject: [PATCH 187/227] Move 'nix hash-*' and 'nix to-*' to 'nix hash' From the 'nix' UX review. --- src/nix/hash.cc | 52 +++++++++++++++++++++++++++++++++------------- tests/brotli.sh | 4 ++-- tests/fetchurl.sh | 8 +++---- tests/hash.sh | 10 ++++----- tests/pure-eval.sh | 2 +- tests/tarball.sh | 2 +- 6 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 7f3d5717a..101b67e6a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdHash : Command +struct CmdHashBase : Command { FileIngestionMethod mode; Base base = SRI; @@ -17,7 +17,7 @@ struct CmdHash : Command std::vector paths; std::optional modulus; - CmdHash(FileIngestionMethod mode) : mode(mode) + CmdHashBase(FileIngestionMethod mode) : mode(mode) { mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); @@ -51,8 +51,6 @@ struct CmdHash : Command return d; } - Category category() override { return catUtility; } - void run() override { for (auto path : paths) { @@ -79,9 +77,6 @@ struct CmdHash : Command } }; -static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); -static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); - struct CmdToBase : Command { Base base; @@ -103,8 +98,6 @@ struct CmdToBase : Command "SRI"); } - Category category() override { return catUtility; } - void run() override { for (auto s : args) @@ -112,10 +105,41 @@ struct CmdToBase : Command } }; -static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref(SRI); }); +struct CmdHash : NixMultiCommand +{ + CmdHash() + : MultiCommand({ + {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, + {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, + {"to-base16", []() { return make_ref(Base16); }}, + {"to-base32", []() { return make_ref(Base32); }}, + {"to-base64", []() { return make_ref(Base64); }}, + {"to-sri", []() { return make_ref(SRI); }}, + }) + { } + + std::string description() override + { + return "compute and convert cryptographic hashes"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix hash' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -149,7 +173,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; diff --git a/tests/brotli.sh b/tests/brotli.sh index a3c6e55a8..dc9bbdb66 100644 --- a/tests/brotli.sh +++ b/tests/brotli.sh @@ -9,13 +9,13 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to $cacheURI $outPath -HASH=$(nix hash-path $outPath) +HASH=$(nix hash path $outPath) clearStore clearCacheCache nix copy --from $cacheURI $outPath --no-check-sigs -HASH2=$(nix hash-path $outPath) +HASH2=$(nix hash path $outPath) [[ $HASH = $HASH2 ]] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 0f2044342..89ff08d4d 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -12,7 +12,7 @@ cmp $outPath fetchurl.sh # Now using a base-64 hash. clearStore -hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) +hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh) outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) @@ -21,7 +21,7 @@ cmp $outPath fetchurl.sh # Now using an SRI hash. clearStore -hash=$(nix hash-file ./fetchurl.sh) +hash=$(nix hash file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] @@ -34,14 +34,14 @@ clearStore other_store=file://$TEST_ROOT/other_store?store=/fnord/store -hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) +hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ --no-out-link --substituters $other_store # Test unpacking a NAR. diff --git a/tests/hash.sh b/tests/hash.sh index 4cfc97901..e5f75e2cf 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -2,7 +2,7 @@ source common.sh try () { printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) + hash=$(nix hash file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) if test "$hash" != "$3"; then echo "hash $1, expected $3, got $hash" exit 1 @@ -69,17 +69,17 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion. try3() { - h64=$(nix to-base64 --type "$1" "$2") + h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] - sri=$(nix to-sri --type "$1" "$2") + sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] - h16=$(nix to-base16 --type "$1" "$h64") + h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] - h16=$(nix to-base16 "$sri") + h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 561ca53fb..c994fbb98 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -15,7 +15,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") -nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x" +nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x" rm -rf $TEST_ROOT/eval-out nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' diff --git a/tests/tarball.sh b/tests/tarball.sh index fe65a22e4..d53ec8cd9 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -10,7 +10,7 @@ mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ -hash=$(nix hash-path $tarroot) +hash=$(nix hash path $tarroot) test_tarball() { local ext="$1" From 5781f45c46d7f6c55f87528ab1e754d070bc99ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 17:55:27 +0100 Subject: [PATCH 188/227] Allow registering subcommands of subcommands --- src/nix/command.cc | 16 +++++++++++++++- src/nix/command.hh | 13 +++++++++++-- src/nix/main.cc | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 9a38c77f1..596217775 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -11,7 +11,21 @@ extern char * * environ __attribute__((weak)); namespace nix { -Commands * RegisterCommand::commands = nullptr; +RegisterCommand::Commands * RegisterCommand::commands = nullptr; + +nix::Commands RegisterCommand::getCommandsFor(const std::vector & prefix) +{ + nix::Commands res; + for (auto & [name, command] : *RegisterCommand::commands) + if (name.size() == prefix.size() + 1) { + bool equal = true; + for (size_t i = 0; i < prefix.size(); ++i) + if (name[i] != prefix[i]) equal = false; + if (equal) + res.insert_or_assign(name[prefix.size()], command); + } + return res; +} void NixMultiCommand::printHelp(const string & programName, std::ostream & out) { diff --git a/src/nix/command.hh b/src/nix/command.hh index d60c8aeb6..6882db195 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -176,20 +176,29 @@ struct StorePathCommand : public InstallablesCommand /* A helper class for registering commands globally. */ struct RegisterCommand { + typedef std::map, std::function()>> Commands; static Commands * commands; - RegisterCommand(const std::string & name, + RegisterCommand(std::vector && name, std::function()> command) { if (!commands) commands = new Commands; commands->emplace(name, command); } + + static nix::Commands getCommandsFor(const std::vector & prefix); }; template static RegisterCommand registerCommand(const std::string & name) { - return RegisterCommand(name, [](){ return make_ref(); }); + return RegisterCommand({name}, [](){ return make_ref(); }); +} + +template +static RegisterCommand registerCommand2(std::vector && name) +{ + return RegisterCommand(std::move(name), [](){ return make_ref(); }); } Buildables build(ref store, Realise mode, diff --git a/src/nix/main.cc b/src/nix/main.cc index a75f8ae65..6d8c66406 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,7 +59,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs bool useNet = true; bool refresh = false; - NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[Command::catDefault] = "Main commands"; From 79c1967ded92574129c6a20116ef205a9c747bac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 18:06:46 +0100 Subject: [PATCH 189/227] Introduce 'nix store' command --- src/nix/store.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/nix/store.cc diff --git a/src/nix/store.cc b/src/nix/store.cc new file mode 100644 index 000000000..e91bcc503 --- /dev/null +++ b/src/nix/store.cc @@ -0,0 +1,31 @@ +#include "command.hh" + +using namespace nix; + +struct CmdStore : virtual NixMultiCommand +{ + CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + { } + + std::string description() override + { + return "manipulate a Nix store"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix store' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdStore = registerCommand("store"); From ef583303f0720d8bc9d6351cd769f92d5dd678f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:42:24 +0200 Subject: [PATCH 190/227] Move NAR-related commands to 'nix nar' --- src/nix/cat.cc | 6 ++---- src/nix/ls.cc | 8 +++----- src/nix/nar.cc | 31 +++++++++++++++++++++++++++++++ tests/binary-cache.sh | 6 +++--- tests/linux-sandbox.sh | 4 ++-- tests/nar-access.sh | 26 +++++++++++++------------- 6 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 src/nix/nar.cc diff --git a/src/nix/cat.cc b/src/nix/cat.cc index eef172cfc..4fa1c9491 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -64,13 +64,11 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } - Category category() override { return catUtility; } - void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static auto rCmdCatStore = registerCommand("cat-store"); -static auto rCmdCatNar = registerCommand("cat-nar"); +static auto rCmdCatStore = registerCommand2({"store", "cat"}); +static auto rCmdCatNar = registerCommand2({"nar", "cat"}); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index f39fdb2fd..d5fec4d84 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -134,7 +134,7 @@ struct CmdLsNar : Command, MixLs return { Example{ "To list a specific file in a NAR:", - "nix ls-nar -l hello.nar /bin/hello" + "nix nar ls -l hello.nar /bin/hello" }, }; } @@ -144,13 +144,11 @@ struct CmdLsNar : Command, MixLs return "show information about a path inside a NAR file"; } - Category category() override { return catUtility; } - void run() override { list(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static auto rCmdLsStore = registerCommand("ls-store"); -static auto rCmdLsNar = registerCommand("ls-nar"); +static auto rCmdLsStore = registerCommand2({"store", "ls"}); +static auto rCmdLsNar = registerCommand2({"nar", "ls"}); diff --git a/src/nix/nar.cc b/src/nix/nar.cc new file mode 100644 index 000000000..e239ce96a --- /dev/null +++ b/src/nix/nar.cc @@ -0,0 +1,31 @@ +#include "command.hh" + +using namespace nix; + +struct CmdNar : NixMultiCommand +{ + CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + { } + + std::string description() override + { + return "query the contents of NAR files"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix nar' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdNar = registerCommand("nar"); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index e3b3982fe..8cb17caf8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -196,13 +196,13 @@ narCache=$TEST_ROOT/nar-cache rm -rf $narCache mkdir $narCache -[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] -(! nix cat-store --store file://$cacheDir $outPath/foobar) +(! nix store cat --store file://$cacheDir $outPath/foobar) # Test NAR listing generation. diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 16abd974c..70a90a907 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -22,9 +22,9 @@ outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) nix path-info -r $outPath | grep input-2 -nix ls-store -R -l $outPath | grep foobar +nix store ls -R -l $outPath | grep foobar -nix cat-store $outPath/foobar | grep FOOBAR +nix store cat $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store diff --git a/tests/nar-access.sh b/tests/nar-access.sh index 88b997ca6..dcc2e8a36 100644 --- a/tests/nar-access.sh +++ b/tests/nar-access.sh @@ -9,45 +9,45 @@ cd "$TEST_ROOT" narFile="$TEST_ROOT/path.nar" nix-store --dump $storePath > $narFile -# Check that find and ls-nar match. +# Check that find and nar ls match. ( cd $storePath; find . | sort ) > files.find -nix ls-nar -R -d $narFile "" | sort > files.ls-nar +nix nar ls -R -d $narFile "" | sort > files.ls-nar diff -u files.find files.ls-nar # Check that file contents of data match. -nix cat-nar $narFile /foo/data > data.cat-nar +nix nar cat $narFile /foo/data > data.cat-nar diff -u data.cat-nar $storePath/foo/data # Check that file contents of baz match. -nix cat-nar $narFile /foo/baz > baz.cat-nar +nix nar cat $narFile /foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz -nix cat-store $storePath/foo/baz > baz.cat-nar +nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz # Test --json. diff -u \ - <(nix ls-nar --json $narFile / | jq -S) \ + <(nix nar ls --json $narFile / | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix ls-nar --json -R $narFile /foo | jq -S) \ + <(nix nar ls --json -R $narFile /foo | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) diff -u \ - <(nix ls-nar --json -R $narFile /foo/bar | jq -S) \ + <(nix nar ls --json -R $narFile /foo/bar | jq -S) \ <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) diff -u \ - <(nix ls-store --json $storePath | jq -S) \ + <(nix store ls --json $storePath | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix ls-store --json -R $storePath/foo | jq -S) \ + <(nix store ls --json -R $storePath/foo | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) diff -u \ - <(nix ls-store --json -R $storePath/foo/bar| jq -S) \ + <(nix store ls --json -R $storePath/foo/bar| jq -S) \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' -nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist' +nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. if nix-store --dump $storePath >/dev/full ; then From 0c15ae5d4b3366a14bca885e02599a941a334920 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:44:31 +0200 Subject: [PATCH 191/227] Add FIXME --- src/nix/show-derivation.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 6d4f295d7..8e1a58ac2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -1,4 +1,5 @@ // FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? #include "command.hh" #include "common-args.hh" From af373c2ece2c14ac652313a6f370dc344c85f86e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 22:45:44 +0100 Subject: [PATCH 192/227] Add deprecated aliases for renamed commands --- src/libutil/args.cc | 5 +---- src/libutil/args.hh | 5 +++-- src/nix/main.cc | 30 +++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8bd9c8aeb..61f9503ec 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -86,6 +86,7 @@ void Args::parseCmdline(const Strings & _cmdline) throw UsageError("unrecognised flag '%1%'", arg); } else { + pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) pendingArgs.clear(); @@ -390,10 +391,6 @@ MultiCommand::MultiCommand(const Commands & commands) .optional = true, .handler = {[=](std::string s) { assert(!command); - if (auto alias = get(deprecatedAliases, s)) { - warn("'%s' is a deprecated alias for '%s'", s, *alias); - s = *alias; - } if (auto prefix = needsCompletion(s)) { for (auto & [name, command] : commands) if (hasPrefix(name, *prefix)) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 26f1bc11b..8069fd70f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -115,6 +115,9 @@ protected: virtual bool processArgs(const Strings & args, bool finish); + virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) + { return pos; } + std::set hiddenCategories; public: @@ -257,8 +260,6 @@ public: std::map categories; - std::map deprecatedAliases; - // Selected command, if any. std::optional>> command; diff --git a/src/nix/main.cc b/src/nix/main.cc index 6d8c66406..94f4cad3c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -112,8 +112,36 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "consider all previously downloaded files out-of-date", .handler = {[&]() { refresh = true; }}, }); + } - deprecatedAliases.insert({"dev-shell", "develop"}); + std::map> aliases = { + {"dev-shell", {"develop"}}, + {"hash-file", {"hash", "file"}}, + {"hash-path", {"hash", "path"}}, + {"to-base16", {"hash", "to-base16"}}, + {"to-base32", {"hash", "to-base32"}}, + {"to-base64", {"hash", "to-base64"}}, + {"ls-nar", {"nar", "ls"}}, + {"ls-store", {"store", "ls"}}, + {"cat-nar", {"nar", "cat"}}, + {"cat-store", {"store", "cat"}}, + }; + + bool aliasUsed = false; + + Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override + { + if (aliasUsed || command || pos == args.end()) return pos; + auto arg = *pos; + auto i = aliases.find(arg); + if (i == aliases.end()) return pos; + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", i->second)); + pos = args.erase(pos); + for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) + pos = args.insert(pos, *j); + aliasUsed = true; + return pos; } void printFlags(std::ostream & out) override From a1cd805cba7a4408e75779bc4099f92e81fd6ac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 21:04:26 +0200 Subject: [PATCH 193/227] Add 'nix nar dump-path' This only differs from 'nix store dump-path' in that the path doesn't need to be a store path. --- src/nix/dump-path.cc | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/nix/main.cc | 1 + 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 6fd197531..4b225ae9f 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "archive.hh" using namespace nix; @@ -7,7 +8,7 @@ struct CmdDumpPath : StorePathCommand { std::string description() override { - return "dump a store path to stdout (in NAR format)"; + return "serialise a store path to stdout in NAR format"; } Examples examples() override @@ -30,4 +31,43 @@ struct CmdDumpPath : StorePathCommand } }; -static auto rDumpPath = registerCommand("dump-path"); + +static auto rDumpPath = registerCommand2({"store", "dump-path"}); + +struct CmdDumpPath2 : Command +{ + Path path; + + CmdDumpPath2() + { + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); + } + + std::string description() override + { + return "serialise a path to stdout in NAR format"; + } + + Examples examples() override + { + return { + Example{ + "To serialise directory 'foo' as a NAR:", + "nix nar dump-path ./foo" + }, + }; + } + + void run() override + { + FdSink sink(STDOUT_FILENO); + dumpPath(path, sink); + sink.flush(); + } +}; + +static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/main.cc b/src/nix/main.cc index 94f4cad3c..0002be291 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -125,6 +125,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"ls-store", {"store", "ls"}}, {"cat-nar", {"nar", "cat"}}, {"cat-store", {"store", "cat"}}, + {"dump-path", {"store", "dump-path"}}, }; bool aliasUsed = false; From ea2062a2d9144d78588675950fc04756f0d200a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:38:56 +0200 Subject: [PATCH 194/227] Move most store-related commands to 'nix store' --- src/nix/add-to-store.cc | 4 +-- src/nix/cat.cc | 2 -- src/nix/diff-closures.cc | 6 ++-- src/nix/dump-path.cc | 5 +-- src/nix/ls.cc | 4 +-- src/nix/main.cc | 18 ++++++++--- src/nix/make-content-addressable.cc | 8 ++--- src/nix/optimise-store.cc | 6 ++-- src/nix/ping-store.cc | 6 ++-- src/nix/sigs.cc | 8 ++--- src/nix/verify.cc | 8 ++--- tests/binary-cache.sh | 2 +- tests/fetchurl.sh | 2 +- tests/gc-auto.sh | 6 ++-- tests/recursive.sh | 4 +-- tests/signing.sh | 48 ++++++++++++++--------------- tests/ssh-relay.sh | 2 +- 17 files changed, 62 insertions(+), 77 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index a2721431e..66822b0ff 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,8 +43,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand ; } - Category category() override { return catUtility; } - void run(ref store) override { if (!namePart) namePart = baseNameOf(path); @@ -80,4 +78,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto rCmdAddToStore = registerCommand("add-to-store"); +static auto rCmdAddToStore = registerCommand2({"store", "add-path"}); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 4fa1c9491..2ecffc9a5 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -37,8 +37,6 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } - Category category() override { return catUtility; } - void run(ref store) override { cat(store->getFSAccessor()); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 30e7b20e1..f72b5eff7 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -121,14 +121,12 @@ struct CmdDiffClosures : SourceExprCommand return "show what packages and versions were added and removed between two closures"; } - Category category() override { return catSecondary; } - Examples examples() override { return { { "To show what got added and removed between two versions of the NixOS system profile:", - "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", + "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", }, }; } @@ -143,4 +141,4 @@ struct CmdDiffClosures : SourceExprCommand } }; -static auto rCmdDiffClosures = registerCommand("diff-closures"); +static auto rCmdDiffClosures = registerCommand2({"store", "diff-closures"}); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 4b225ae9f..256db64a9 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -16,13 +16,11 @@ struct CmdDumpPath : StorePathCommand return { Example{ "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" }, }; } - Category category() override { return catUtility; } - void run(ref store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); @@ -31,7 +29,6 @@ struct CmdDumpPath : StorePathCommand } }; - static auto rDumpPath = registerCommand2({"store", "dump-path"}); struct CmdDumpPath2 : Command diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d5fec4d84..1f5ed6913 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -97,7 +97,7 @@ struct CmdLsStore : StoreCommand, MixLs return { Example{ "To list the contents of a store path in a binary cache:", - "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" }, }; } @@ -107,8 +107,6 @@ struct CmdLsStore : StoreCommand, MixLs return "show information about a path in the Nix store"; } - Category category() override { return catUtility; } - void run(ref store) override { list(store->getFSAccessor()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 0002be291..fb3bffeaf 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -115,17 +115,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs } std::map> aliases = { + {"add-to-store", {"store", "add-path"}}, + {"cat-nar", {"nar", "cat"}}, + {"cat-store", {"store", "cat"}}, + {"copy-sigs", {"store", "copy-sigs"}}, {"dev-shell", {"develop"}}, + {"diff-closures", {"store", "diff-closures"}}, + {"dump-path", {"store", "dump-path"}}, {"hash-file", {"hash", "file"}}, {"hash-path", {"hash", "path"}}, + {"ls-nar", {"nar", "ls"}}, + {"ls-store", {"store", "ls"}}, + {"make-content-addressable", {"store", "make-content-addressable"}}, + {"optimise-store", {"store", "optimise"}}, + {"ping-store", {"store", "ping"}}, + {"sign-paths", {"store", "sign-paths"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, - {"ls-nar", {"nar", "ls"}}, - {"ls-store", {"store", "ls"}}, - {"cat-nar", {"nar", "cat"}}, - {"cat-store", {"store", "cat"}}, - {"dump-path", {"store", "dump-path"}}, + {"verify", {"store", "verify"}}, }; bool aliasUsed = false; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 12c2cf776..0dade90ef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -23,17 +23,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON return { Example{ "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix make-content-addressable nixpkgs#hello" + "nix store make-content-addressable nixpkgs#hello" }, Example{ "To compute a content-addressable representation of the current NixOS system closure:", - "nix make-content-addressable -r /run/current-system" + "nix store make-content-addressable -r /run/current-system" }, }; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); @@ -108,4 +106,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto rCmdMakeContentAddressable = registerCommand("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand2({"store", "make-content-addressable"}); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index 51a7a9756..bc7f175ac 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -18,17 +18,15 @@ struct CmdOptimiseStore : StoreCommand return { Example{ "To optimise the Nix store:", - "nix optimise-store" + "nix store optimise" }, }; } - Category category() override { return catUtility; } - void run(ref store) override { store->optimiseStore(); } }; -static auto rCmdOptimiseStore = registerCommand("optimise-store"); +static auto rCmdOptimiseStore = registerCommand2({"store", "optimise"}); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 8db78d591..19b1a55c8 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -16,17 +16,15 @@ struct CmdPingStore : StoreCommand return { Example{ "To test whether connecting to a remote Nix store via SSH works:", - "nix ping-store --store ssh://mac1" + "nix store ping --store ssh://mac1" }, }; } - Category category() override { return catUtility; } - void run(ref store) override { store->connect(); } }; -static auto rCmdPingStore = registerCommand("ping-store"); +static auto rCmdPingStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 44916c77f..37b8a6712 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,8 +27,6 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -92,7 +90,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto rCmdCopySigs = registerCommand("copy-sigs"); +static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); struct CmdSignPaths : StorePathsCommand { @@ -115,8 +113,6 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { if (secretKeyFile.empty()) @@ -144,4 +140,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand("sign-paths"); +static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ec7333d03..bcf85d7dd 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -40,17 +40,15 @@ struct CmdVerify : StorePathsCommand return { Example{ "To verify the entire Nix store:", - "nix verify --all" + "nix store verify --all" }, Example{ "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix verify -r -n2 --no-contents $(type -p firefox)" + "nix store verify -r -n2 --no-contents $(type -p firefox)" }, }; } - Category category() override { return catSecondary; } - void run(ref store, StorePaths storePaths) override { std::vector> substituters; @@ -189,4 +187,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto rCmdVerify = registerCommand("verify"); +static auto rCmdVerify = registerCommand2({"store", "verify"}); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 8cb17caf8..92ed36225 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -188,7 +188,7 @@ unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store file://$cacheDir --no-trust # Test local NAR caching. diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 89ff08d4d..7ec25808c 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) +storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 3add896c6..6867f2eb4 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -2,9 +2,9 @@ source common.sh clearStore -garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) -garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) -garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) +garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) +garbage2=$(nix store add-path --name garbage2 ./nar-access.sh) +garbage3=$(nix store add-path --name garbage3 ./nar-access.sh) ls -l $garbage3 POSIXLY_CORRECT=1 du $garbage3 diff --git a/tests/recursive.sh b/tests/recursive.sh index 80a178cc7..b020ec710 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -7,7 +7,7 @@ clearStore rm -f $TEST_ROOT/result -export unreachable=$(nix add-to-store ./recursive.sh) +export unreachable=$(nix store add-path ./recursive.sh) NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' with import ./config.nix; @@ -38,7 +38,7 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r # Add something to the store. echo foobar > foobar - foobar=$(nix $opts add-to-store ./foobar) + foobar=$(nix $opts store add-path ./foobar) nix $opts path-info $foobar nix $opts build $foobar diff --git a/tests/signing.sh b/tests/signing.sh index 9e29e3fbf..bd6280cc6 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -17,40 +17,40 @@ info=$(nix path-info --json $outPath) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] -# Test "nix verify". -nix verify -r $outPath +# Test "nix store verify". +nix store verify -r $outPath -expect 2 nix verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r $outPath --sigs-needed 1 -nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 -expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 -nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" -nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix verify -r $outPath +nix store verify -r $outPath # Verify that the path did not get signed but does have the ultimate bit. info=$(nix path-info --json $outPath2) [[ $info =~ '"ultimate":true' ]] (! [[ $info =~ 'signatures' ]]) -# Test "nix verify". -nix verify -r $outPath2 +# Test "nix store verify". +nix store verify -r $outPath2 -expect 2 nix verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r $outPath2 --sigs-needed 1 -expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix sign-paths". -nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign-paths". +nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 -nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) @@ -59,12 +59,12 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix verify $outPathCA -nix verify $outPathCA --sigs-needed 1000 +nix store verify $outPathCA +nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA -nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. nix copy --to file://$cacheDir $outPath2 @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] @@ -89,17 +89,17 @@ rm -rf $TEST_ROOT/store0 # But succeed if we supply the public keys. nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 -expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath -nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 +nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. (! nix copy --to $TEST_ROOT/store0 $outPath2) nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 # But signatures should still get copied. -nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 # Content-addressed stuff can be copied without signatures. nix copy --to $TEST_ROOT/store0 $outPathCA diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh index dce50974b..053b2f00d 100644 --- a/tests/ssh-relay.sh +++ b/tests/ssh-relay.sh @@ -11,6 +11,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) [ foo = $(< $out) ] From fa8dad10edcebd942ce99e6413fa38e3fe883f15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 23:26:23 +0100 Subject: [PATCH 195/227] Typo --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 75426b2e3..8cf5ccd62 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -413,7 +413,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile return { Example{ "To show what changed between each generation of the NixOS system profile:", - "nix profile diff-closure --profile /nix/var/nix/profiles/system" + "nix profile diff-closures --profile /nix/var/nix/profiles/system" }, }; } From f337aa70998141ccfaa956e9f670152dbb15b385 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Dec 2020 00:58:09 +0100 Subject: [PATCH 196/227] Split 'nix store add-to-store' into 'add-path' and 'add-file' This makes it consistent with 'nix hash '. --- src/nix/{add-to-store.md => add-file.md} | 6 +-- src/nix/add-path.md | 29 +++++++++++ src/nix/add-to-store.cc | 65 ++++++++++++++++-------- tests/fetchurl.sh | 2 +- 4 files changed, 77 insertions(+), 25 deletions(-) rename src/nix/{add-to-store.md => add-file.md} (72%) create mode 100644 src/nix/add-path.md diff --git a/src/nix/add-to-store.md b/src/nix/add-file.md similarity index 72% rename from src/nix/add-to-store.md rename to src/nix/add-file.md index 593ad67ad..ed237a035 100644 --- a/src/nix/add-to-store.md +++ b/src/nix/add-file.md @@ -2,8 +2,8 @@ R""( # Description -Copy the file or directory *path* to the Nix store, and -print the resulting store path on standard output. +Copy the regular file *path* to the Nix store, and print the resulting +store path on standard output. > **Warning** > @@ -18,7 +18,7 @@ Add a regular file to the store: ```console # echo foo > bar -# nix add-to-store --flat ./bar +# nix store add-file ./bar /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar diff --git a/src/nix/add-path.md b/src/nix/add-path.md new file mode 100644 index 000000000..87473611d --- /dev/null +++ b/src/nix/add-path.md @@ -0,0 +1,29 @@ +R""( + +# Description + +Copy *path* to the Nix store, and print the resulting store path on +standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a directory to the store: + +```console +# mkdir dir +# echo foo > dir/bar + +# nix store add-path ./dir +/nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir + +# cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar +foo +``` + +)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 66822b0ff..ea4bbbab9 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,10 +9,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; + FileIngestionMethod ingestionMethod; CmdAddToStore() { + // FIXME: completion expectArg("path", &path); addFlag({ @@ -22,25 +23,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); - - addFlag({ - .longName = "flat", - .shortName = 0, - .description = "add flat file to the Nix store", - .handler = {&ingestionMethod, FileIngestionMethod::Flat}, - }); - } - - std::string description() override - { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-to-store.md" - ; } void run(ref store) override @@ -78,4 +60,45 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto rCmdAddToStore = registerCommand2({"store", "add-path"}); +struct CmdAddFile : CmdAddToStore +{ + CmdAddFile() + { + ingestionMethod = FileIngestionMethod::Flat; + } + + std::string description() override + { + return "add a regular file to the Nix store"; + } + + std::string doc() override + { + return + #include "add-file.md" + ; + } +}; + +struct CmdAddPath : CmdAddToStore +{ + CmdAddPath() + { + ingestionMethod = FileIngestionMethod::Recursive; + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + std::string doc() override + { + return + #include "add-path.md" + ; + } +}; + +static auto rCmdAddFile = registerCommand2({"store", "add-file"}); +static auto rCmdAddPath = registerCommand2({"store", "add-path"}); diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 7ec25808c..10ec0173a 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh) +storePath=$(nix --store $other_store store add-file ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) From be09af80026b751569e7dd6519d1564e10e4f34b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 18:03:30 -0600 Subject: [PATCH 197/227] Include static "nix" binary in Hydra build products This allows users to get Nix from Hydra via a stable url like https://hydra.nixos.org/build/132078238/download/1/nix --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 7311a2471..9addccd63 100644 --- a/flake.nix +++ b/flake.nix @@ -466,6 +466,8 @@ postInstall = '' mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products ''; doInstallCheck = true; From 5f66edf24503fabf410bb923de761131cea57771 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 4 Dec 2020 14:28:27 +0100 Subject: [PATCH 198/227] Make `make install` less noisy Remove the printing and useless output of a couple of commands when running `make install` --- doc/manual/local.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index ee5b328de..81a7755e8 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -68,14 +68,14 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/builtins.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp - mv $@.tmp $@ + @mv $@.tmp $@ # Generate the HTML manual. install: $(docdir)/manual/index.html # Generate 'nix' manpages. install: $(d)/src/command-ref/new-cli - for i in doc/manual/src/command-ref/new-cli/*.md; do \ + $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ name=$$(basename $$i .md); \ if [[ $$name = SUMMARY ]]; then continue; fi; \ printf "Title: %s\n\n" "$$name" > $$i.tmp; \ @@ -84,7 +84,7 @@ install: $(d)/src/command-ref/new-cli done $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md - $(trace-gen) mdbook build doc/manual -d $(docdir)/manual + $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual @cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js endif From e20a3ec756c61b076d5fbaea6d976c8ee7fa4c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Kochen?= Date: Fri, 4 Dec 2020 19:32:35 +0100 Subject: [PATCH 199/227] Fix compatibility with newer AWS SDKs Tested against AWS SDK 1.8.99. Fixes #3201. --- configure.ac | 1 + src/libstore/s3-binary-cache-store.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 39306b953..daf378997 100644 --- a/configure.ac +++ b/configure.ac @@ -255,6 +255,7 @@ if test -n "$enable_s3"; then declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) + AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) fi diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4519dd5b5..27253fc12 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -57,6 +57,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { debug("AWS: %s", chomp(statement)); } + +#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) + void Flush() override {} +#endif }; static void initAWS() From 692549c542f673015105a3aa4358c1e3095bb0e0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:28:09 -0600 Subject: [PATCH 200/227] Use com.apple.oahd.plist for rosetta 2 detection --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 9c8831d2b..0eaf25bb3 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -47,7 +47,7 @@ case "$(uname -s).$(uname -m)" in ;; Darwin.arm64|Darwin.aarch64) # check for Rosetta 2 support - if ! [ -d /Library/Apple/usr/libexec/oah ]; then + if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" fi From b9a00fd15bb158c293b40aec49c9a426cc4c8921 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 22:17:19 -0600 Subject: [PATCH 201/227] =?UTF-8?q?Canonicalize=20binary=20caches=20with?= =?UTF-8?q?=20=E2=80=98/=E2=80=99=20when=20one=20is=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This checks if there is a trusted substituter with a slash, so trusting https://cache.nixos.org also implies https://cache.nixos.org/ is trusted. --- src/libstore/daemon.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e5cfe94cb..2224d54d5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -215,6 +215,8 @@ struct ClientSettings for (auto & s : ss) if (trusted.count(s)) subs.push_back(s); + else if (!hasSuffix(s, "/") && trusted.count(s + "/")) + subs.push_back(s + "/"); else warn("ignoring untrusted substituter '%s'", s); res = subs; From aa07502009625fe0d38fde1a23c50dd34f1996eb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 6 Dec 2020 23:04:42 -0600 Subject: [PATCH 202/227] Always default to cache.nixos.org even when different nix store dir Since 0744f7f, it is now useful to have cache.nixos.org in substituers even if /nix/store is not the Nix Store Dir. This can always be overridden via configuration, though. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..6b4775683 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -583,7 +583,7 @@ public: Setting substituters{ this, - nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + Strings{"https://cache.nixos.org/"}, "substituters", R"( A list of URLs of substituters, separated by whitespace. The default From 0d7714b0d7f626afc174f1faaf52725290a1247c Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 8 Dec 2020 10:25:03 +0100 Subject: [PATCH 203/227] forgot to add the files --- doc/manual/src/command-ref/nix.md | 1 + doc/manual/src/contributing/contributing.md | 0 2 files changed, 1 insertion(+) create mode 100644 doc/manual/src/command-ref/nix.md create mode 100644 doc/manual/src/contributing/contributing.md diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md new file mode 100644 index 000000000..cc677d127 --- /dev/null +++ b/doc/manual/src/command-ref/nix.md @@ -0,0 +1 @@ +# nix diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md new file mode 100644 index 000000000..e69de29bb From 1b1e0760335832c87516b9103b670b34662d5daf Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 11:11:02 +0100 Subject: [PATCH 204/227] Re-query for the derivation outputs in the post-build-hook We can't assume that the runtime state knows about them as they might have been built remotely, in which case we must query the db again to get them. --- src/libstore/build/derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..fdf777c27 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,8 +899,10 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); + for (auto& [_, maybeOutPath] : + worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (maybeOutPath) + outputPaths.insert(*maybeOutPath); } std::map hookEnvironment = getEnv(); From ae77f21474594fc4555bafa3291424b12ff3d4ac Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 8 Dec 2020 11:59:23 +0100 Subject: [PATCH 205/227] Switch away from classification as Tier1-3 to classification to a more descriptive classification. --- doc/manual/src/contributing/cli-guideline.md | 44 ++++++++++++++------ doc/manual/src/contributing/contributing.md | 1 + doc/manual/src/contributing/hacking.md | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 doc/manual/src/contributing/hacking.md diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 01df07136..0132867c8 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -58,12 +58,13 @@ them. This classification tries to separate commands in 3 categories in terms of their importance in regards to the new users. Users who are likely to be -impacted the most by bad user experience. This does not mean that we will not -take care of Tier3 commands, it means we will only put more helpful details for -beginners into Tier1 commands. +impacted the most by bad user experience. + +- **Main commands** + + Commands used for our main use cases and most likely used by new users. We + expect attention to details, such as: -- **Tier1**: Commands used for our main use cases and most likely used by new - users. From Tier1 commands we expect attention to details, such as: - Proper use of [colors](#colors), [emojis](#special-unicode-characters) and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. @@ -73,16 +74,32 @@ beginners into Tier1 commands. Nix ecosystem. - [Help pages](#help-is-essential) to be as good as we can write them pointing to external documentation and tutorials for more. -- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really - exposing some implementation detail, but not something that we expect a user. - From Tier2 command we expect less attention to details, but still some: + + Examples of such commands: `nix init`, `nix develop`, `nix build`, `nix run`, + ... + +- **Infrequently used commands** + + From infrequently used commands we expect less attention to details, but + still some: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. -- **Tier3**: Commands that expose certain internal functionality of `nix`, - mostly used by other scripts. + + Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + +- **Utility and scripting commands** + + Commands that expose certain internal functionality of `nix`, mostly used by + other scripts. + - [Autocomplete](#shell-completion) of options. + Examples of such commands: `nix store copy`, `nix hash base16`, `nix store + ping`, ... + + # Help is essential Help should be built into your command line so that new users can gradually @@ -249,9 +266,10 @@ Here are few examples of flag `OPTIONS`: ## Prompt when input not provided -For **Tier1** commands we want command to improve the discoverability of -possible input. A new user will most likely not know which `ARGUMENTS` and -`OPTIONS` are required or which values are possible for those options. +For *main commands* (as [per classification](#classification)) we want command +to improve the discoverability of possible input. A new user will most likely +not know which `ARGUMENTS` and `OPTIONS` are required or which values are +possible for those options. In cases, the user might not provide the input or they provide wrong input, rather then show the error, prompt a user with an option to find and select diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md index e69de29bb..854139a31 100644 --- a/doc/manual/src/contributing/contributing.md +++ b/doc/manual/src/contributing/contributing.md @@ -0,0 +1 @@ +# Contributing diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md new file mode 100644 index 000000000..2ad773dea --- /dev/null +++ b/doc/manual/src/contributing/hacking.md @@ -0,0 +1 @@ +# Hacking From c0f21f08f817745fcf3e9301749b7e237634521c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 06:57:31 +0100 Subject: [PATCH 206/227] Hide the sqlite statements declarations for the local store These have no need to be in the public interface and it causes spurious rebuilds each time one wants to add or remove a new statement. --- src/libstore/local-store.cc | 76 ++++++++++++++++++++++--------------- src/libstore/local-store.hh | 15 +------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 348e5d0d4..2a47b3956 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,6 +42,21 @@ namespace nix { +struct LocalStore::State::Stmts { + /* Some precompiled SQLite statements. */ + SQLiteStmt RegisterValidPath; + SQLiteStmt UpdatePathInfo; + SQLiteStmt AddReference; + SQLiteStmt QueryPathInfo; + SQLiteStmt QueryReferences; + SQLiteStmt QueryReferrers; + SQLiteStmt InvalidatePath; + SQLiteStmt AddDerivationOutput; + SQLiteStmt QueryValidDerivers; + SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryPathFromHashPart; + SQLiteStmt QueryValidPaths; +}; LocalStore::LocalStore(const Params & params) : StoreConfig(params) @@ -60,6 +75,7 @@ LocalStore::LocalStore(const Params & params) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); + state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); @@ -223,31 +239,31 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create(state->db, + state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create(state->db, + state->stmts->UpdatePathInfo.create(state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); - state->stmtAddReference.create(state->db, + state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create(state->db, + state->stmts->QueryPathInfo.create(state->db, "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, + state->stmts->QueryReferences.create(state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - state->stmtQueryReferrers.create(state->db, + state->stmts->QueryReferrers.create(state->db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, + state->stmts->InvalidatePath.create(state->db, "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create(state->db, + state->stmts->AddDerivationOutput.create(state->db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - state->stmtQueryValidDerivers.create(state->db, + state->stmts->QueryValidDerivers.create(state->db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create(state->db, + state->stmts->QueryDerivationOutputs.create(state->db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create(state->db, + state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); + state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); } @@ -590,7 +606,7 @@ void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & out void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { - state.stmtAddDerivationOutput.use() + state.stmts->AddDerivationOutput.use() (deriver) (outputName) (printStorePath(output)) @@ -607,7 +623,7 @@ uint64_t LocalStore::addValidPath(State & state, throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); - state.stmtRegisterValidPath.use() + state.stmts->RegisterValidPath.use() (printStorePath(info.path)) (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) @@ -659,7 +675,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); + auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr(); @@ -679,7 +695,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->registrationTime = useQueryPathInfo.getInt(2); - auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ @@ -687,14 +703,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->ultimate = useQueryPathInfo.getInt(5) == 1; - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); if (s) info->sigs = tokenizeString(s, " "); - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); @@ -709,7 +725,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - state.stmtUpdatePathInfo.use() + state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) @@ -722,7 +738,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); + auto use(state.stmts->QueryPathInfo.use()(printStorePath(path))); if (!use.next()) throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); @@ -731,7 +747,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); + return state.stmts->QueryPathInfo.use()(printStorePath(path)).next(); } @@ -757,7 +773,7 @@ StorePathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); + auto use(state->stmts->QueryValidPaths.use()); StorePathSet res; while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; @@ -767,7 +783,7 @@ StorePathSet LocalStore::queryAllValidPaths() void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); + auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); @@ -788,7 +804,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) return retrySQLite([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); + auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path))); StorePathSet derivers; while (useQueryValidDerivers.next()) @@ -848,7 +864,7 @@ std::map> LocalStore::queryPartialDerivati } auto useQueryDerivationOutputs { - state->stmtQueryDerivationOutputs.use() + state->stmts->QueryDerivationOutputs.use() (drvId) }; @@ -872,11 +888,11 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); + auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix)); if (!useQueryPathFromHashPart.next()) return {}; - const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0); if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) return parseStorePath(s); return {}; @@ -990,7 +1006,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); + state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1030,7 +1046,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(printStorePath(path)).exec(); + state.stmts->InvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 58ec93f27..332718af4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -55,19 +55,8 @@ private: /* The SQLite database object. */ SQLite db; - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - SQLiteStmt stmtQueryValidPaths; + struct Stmts; + std::unique_ptr stmts; /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; From 6758e65612b990805d3d7d2039cd92647730e900 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 09:44:07 +0100 Subject: [PATCH 207/227] Revert "Re-query for the derivation outputs in the post-build-hook" This reverts commit 1b1e0760335832c87516b9103b670b34662d5daf. Using `queryPartialDerivationOutputMap` assumes that the derivation exists locally which isn't the case for remote builders. --- src/libstore/build/derivation-goal.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fdf777c27..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,10 +899,8 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto& [_, maybeOutPath] : - worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (maybeOutPath) - outputPaths.insert(*maybeOutPath); + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); } std::map hookEnvironment = getEnv(); From ee7c94fa1b74b43f7a719373f5f566d09b147126 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 10:37:39 +0100 Subject: [PATCH 208/227] Test the post-build-hook with remote builders Regression test for #4245 --- tests/build-remote-input-addressed.sh | 28 +++++++++++++++++++++++++++ tests/build-remote.sh | 3 +++ 2 files changed, 31 insertions(+) diff --git a/tests/build-remote-input-addressed.sh b/tests/build-remote-input-addressed.sh index b34caa061..49d15c389 100644 --- a/tests/build-remote-input-addressed.sh +++ b/tests/build-remote-input-addressed.sh @@ -3,3 +3,31 @@ source common.sh file=build-hook.nix source build-remote.sh + +# Add a `post-build-hook` option to the nix conf. +# This hook will be executed both for the local machine and the remote builders +# (because they share the same config). +registerBuildHook () { + # Dummy post-build-hook just to ensure that it's executed correctly. + # (we can't reuse the one from `$PWD/push-to-store.sh` because of + # https://github.com/NixOS/nix/issues/4341) + cat < $TEST_ROOT/post-build-hook.sh +#!/bin/sh + +echo "Post hook ran successfully" +# Add an empty line to a counter file, just to check that this hook ran properly +echo "" >> $TEST_ROOT/post-hook-counter +EOF + chmod +x $TEST_ROOT/post-build-hook.sh + rm -f $TEST_ROOT/post-hook-counter + + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf +} + +registerBuildHook +source build-remote.sh + +# `build-hook.nix` has four derivations to build, and the hook runs twice for +# each derivation (once on the builder and once on the host), so the counter +# should contain eight lines now +[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]] diff --git a/tests/build-remote.sh b/tests/build-remote.sh index ca6d1de09..04848e4b5 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -14,6 +14,9 @@ builders=( "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz" ) +chmod -R +w $TEST_ROOT/machine* || true +rm -rf $TEST_ROOT/machine* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). From c87267c2a4621a42d6bfcdb7944cd546f194787a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 10:38:04 +0100 Subject: [PATCH 209/227] Store the final drv outputs in memory when building remotely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `DerivationGoal` has a variable storing the “final” derivation output paths that is used (amongst other things) to fill the environment for the post build hook. However this variable wasn't set when the build-hook is used, causing a crash when both hooks are used together. Fix this by setting this variable (from the informations in the db) after a run of the post build hook. --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..de58d9f06 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2872,6 +2872,8 @@ void DerivationGoal::registerOutputs() for (auto & i : drv->outputsAndOptPaths(worker.store)) { if (!i.second.second || !worker.store.isValidPath(*i.second.second)) allValid = false; + else + finalOutputs.insert_or_assign(i.first, *i.second.second); } if (allValid) return; } From 93a8a005defbe5204f739853403ef11dbc016f33 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 5 Dec 2020 15:33:16 +0100 Subject: [PATCH 210/227] libstore/openStore: fix stores with IPv6 addresses In `nixStable` (2.3.7 to be precise) it's possible to connect to stores using an IPv6 address: nix ping-store --store ssh://root@2001:db8::1 This is also useful for `nixops(1)` where you could specify an IPv6 address in `deployment.targetHost`. However, this behavior is broken on `nixUnstable` and fails with the following error: $ nix store ping --store ssh://root@2001:db8::1 don't know how to open Nix store 'ssh://root@2001:db8::1' This happened because `openStore` from `libstore` uses the `parseURL` function from `libfetchers` which expects a valid URL as defined in RFC2732. However, this is unsupported by `ssh(1)`: $ nix store ping --store 'ssh://root@[2001:db8::1]' cannot connect to 'root@[2001:db8::1]' This patch now allows both ways of specifying a store (`root@2001:db8::1`) and also `root@[2001:db8::1]` since the latter one is useful to pass query parameters to the remote store. In order to achieve this, the following changes were made: * The URL regex from `url-parts.hh` now allows an IPv6 address in the form `2001:db8::1` and also `[2001:db8::1]`. * In `libstore`, a new function named `extractConnStr` ensures that a proper URL is passed to e.g. `ssh(1)`: * If a URL looks like either `[2001:db8::1]` or `root@[2001:db8::1]`, the brackets will be removed using a regex. No additional validation is done here as only strings parsed by `parseURL` are expected. * In any other case, the string will be left untouched. * The rules above only apply for `LegacySSHStore` and `SSHStore` (a.k.a `ssh://` and `ssh-ng://`). Unresolved questions: * I'm not really sure whether we want to allow both variants of IPv6 addresses in the URL parser. However it should be noted that both seem to be possible according to RFC2732: > This document incudes an update to the generic syntax for Uniform > Resource Identifiers defined in RFC 2396 [URL]. It defines a syntax > for IPv6 addresses and allows the use of "[" and "]" within a URI > explicitly for this reserved purpose. * Currently, it's not supported to specify a port number behind the hostname, however it seems as this is not really supported by the URL parser. Hence, this is probably out of scope here. --- src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++++++++- src/libutil/url-parts.hh | 3 ++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 27be66cac..7bf9235b2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,8 @@ #include "archive.hh" #include "callback.hh" +#include + namespace nix { @@ -1091,6 +1093,34 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } } +// The `parseURL` function supports both IPv6 URIs as defined in +// RFC2732, but also pure addresses. The latter one is needed here to +// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). +// +// This function now ensures that a usable connection string is available: +// * If the store to be opened is not an SSH store, nothing will be done. +// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably +// needed to pass further flags), it +// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). +// * If the URL looks like `root@::1` it will be left as-is. +// * In any other case, the string will be left as-is. +static std::string extractConnStr(const std::string &proto, const std::string &connStr) +{ + if (proto.rfind("ssh") != std::string::npos) { + std::smatch result; + std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + + if (std::regex_match(connStr, result, v6AddrRegex)) { + if (result[1].matched) { + return result.str(1) + result.str(3); + } + return result.str(3); + } + } + + return connStr; +} + ref openStore(const std::string & uri_, const Store::Params & extraParams) { @@ -1099,7 +1129,10 @@ ref openStore(const std::string & uri_, auto parsedUri = parseURL(uri_); params.insert(parsedUri.query.begin(), parsedUri.query.end()); - auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + auto baseURI = extractConnStr( + parsedUri.scheme, + parsedUri.authority.value_or("") + parsedUri.path + ); for (auto implem : *Implementations::registered) { if (implem.uriSchemes.count(parsedUri.scheme)) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 68be15cb0..5d21b8d1a 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,8 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+"; +const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; From 5286310e593b2ac13abbbac90c5b458c4be2ad37 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 14:53:45 +0100 Subject: [PATCH 211/227] Use no substituers by default in the tests Otherwise https://cache.nixos.org is chosen by default, causing the OSX testsuite to hang inside the sandbox. (In a way, this is probably rugging an actual bug under the carpet as Nix should be able to gracefully timeout in such a case, but that's beyond mac OSX-fu) --- tests/init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/init.sh b/tests/init.sh index f9ced6b0d..63cf895e2 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -19,6 +19,7 @@ keep-derivations = false sandbox = false experimental-features = nix-command flakes gc-reserved-space = 0 +substituters = flake-registry = $TEST_ROOT/registry.json include nix.conf.extra EOF From a8f533b66417a1025a468cae3068bd2f5c06e811 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Nov 2020 11:19:36 +0100 Subject: [PATCH 212/227] Add lvlNotice log level This is like syslog's LOG_NOTICE: "normal, but significant, condition". --- src/libutil/error.hh | 1 + src/libutil/logging.hh | 1 + src/nix/main.cc | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..aa4fadfcc 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -45,6 +45,7 @@ namespace nix { typedef enum { lvlError = 0, lvlWarn, + lvlNotice, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 82ba54051..96ad69790 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -198,6 +198,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ } while (0) #define printError(args...) printMsg(lvlError, args) +#define notice(args...) printMsg(lvlNotice, args) #define printInfo(args...) printMsg(lvlInfo, args) #define printTalkative(args...) printMsg(lvlTalkative, args) #define debug(args...) printMsg(lvlDebug, args) diff --git a/src/nix/main.cc b/src/nix/main.cc index fb3bffeaf..27b1d7257 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -250,7 +250,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlWarn; + verbosity = lvlNotice; settings.verboseBuild = false; evalSettings.pureEval = true; From c6a1bcd0ec1ed443947ae7151e32dd6827dfe53e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 17:11:39 +0100 Subject: [PATCH 213/227] nix store make-content-addressable: Show rewritten path --- src/nix/make-content-addressable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0dade90ef..5165c4804 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }; if (!json) - printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); From eb453081092cbee5f8176c1d348ac23e46a24281 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 10 Dec 2020 17:40:00 +0100 Subject: [PATCH 214/227] Fix the `nix` command with CA derivations Prevents a crash because most `nix` subcommands assumed that derivations know their output path, which isn't the case for CA derivations --- src/nix/installables.cc | 2 +- tests/content-addressed.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b6ed030af..3506c3fcc 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -409,7 +409,7 @@ std::vector InstallableAttrPath::toDerivations for (auto & drvInfo : drvInfos) { res.push_back({ state->store->parseStorePath(drvInfo.queryDrvPath()), - state->store->parseStorePath(drvInfo.queryOutPath()), + state->store->maybeParseStorePath(drvInfo.queryOutPath()), drvInfo.queryOutputName() }); } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 03eff549c..bc37a99c1 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -50,7 +50,13 @@ testGC () { nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true } +testNixCommand () { + clearStore + nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link +} + testRemoteCache testDeterministicCA testCutoff testGC +testNixCommand From 63b3536f50f124cdcd7592b344eac157a1439d42 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Fri, 28 Aug 2020 10:28:35 -0300 Subject: [PATCH 215/227] treat s3 permission errors as file-not-found Signed-off-by: Jonathan Ringer --- src/libstore/s3-binary-cache-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27253fc12..d6edafd7e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject( dynamic_cast(result.GetBody()).str()); } catch (S3Error & e) { - if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) && + (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw; } auto now2 = std::chrono::steady_clock::now(); From 58cdab64acd4807f73768fb32acdde39b501799f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Oct 2020 17:36:51 +0200 Subject: [PATCH 216/227] Store metadata about drv outputs realisations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each known realisation, store: - its output - its output path This comes with a set of needed changes: - New `realisations` module declaring the types needed for describing these mappings - New `Store::registerDrvOutput` method registering all the needed informations about a derivation output (also replaces `LocalStore::linkDeriverToPath`) - new `Store::queryRealisation` method to retrieve the informations for a derivations This introcudes some redundancy on the remote-store side between `wopQueryDerivationOutputMap` and `wopQueryRealisation`. However we might need to keep both (regardless of backwards compat) because we sometimes need to get some infos for all the outputs of a derivation (where `wopQueryDerivationOutputMap` is handy), but all the stores can't implement it − because listing all the outputs of a derivation isn't really possible for binary caches where the server doesn't allow to list a directory. --- src/libstore/binary-cache-store.cc | 17 ++++++ src/libstore/binary-cache-store.hh | 7 +++ src/libstore/build/derivation-goal.cc | 19 ++++++- src/libstore/daemon.cc | 22 ++++++++ src/libstore/dummy-store.cc | 3 + src/libstore/legacy-ssh-store.cc | 4 ++ src/libstore/local-binary-cache-store.cc | 1 + src/libstore/local-store.cc | 21 +++++-- src/libstore/local-store.hh | 12 ++-- src/libstore/realisation.cc | 72 ++++++++++++++++++++++++ src/libstore/realisation.hh | 34 +++++++++++ src/libstore/remote-store.cc | 21 +++++++ src/libstore/remote-store.hh | 4 ++ src/libstore/store-api.hh | 15 +++++ src/libstore/worker-protocol.hh | 5 ++ tests/content-addressed.sh | 3 +- 16 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 src/libstore/realisation.cc create mode 100644 src/libstore/realisation.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a918b7208..085dc7ba1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -443,6 +443,23 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } +std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) +{ + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; + auto rawOutputInfo = getFile(outputInfoFilePath); + + if (rawOutputInfo) { + return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + } else { + return std::nullopt; + } +} + +void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; + upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); +} + ref BinaryCacheStore::getFSAccessor() { return make_ref(ref(shared_from_this()), localNarCache); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5224d7ec8..07a8b2beb 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -33,6 +33,9 @@ private: protected: + // The prefix under which realisation infos will be stored + const std::string realisationsPrefix = "/realisations"; + BinaryCacheStore(const Params & params); public: @@ -99,6 +102,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void narFromPath(const StorePath & path, Sink & sink) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de58d9f06..a0f10c33d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2094,6 +2094,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf /* Nothing to be done; 'path' must already be valid. */ } + void registerDrvOutput(const Realisation & info) override + { + if (!goal.isAllowed(info.id.drvPath)) + throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + next->registerDrvOutput(info); + } + + std::optional queryRealisation(const DrvOutput & id) override + { + if (!goal.isAllowed(id.drvPath)) + throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + return next->queryRealisation(id); + } + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -3393,7 +3407,10 @@ void DerivationGoal::registerOutputs() if (useDerivation || isCaFloating) for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); + worker.store.registerDrvOutput( + DrvOutputId{drvPathResolved, outputName}, + DrvOutputInfo{.outPath = newInfo.path, + .resolvedDrv = drvPathResolved}); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2224d54d5..ba5788b64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopRegisterDrvOutput: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + auto resolvedDrv = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + logger->stopWork(); + break; + } + + case wopQueryRealisation: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto info = store->queryRealisation(outputId); + logger->stopWork(); + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + break; + } + default: throw Error("invalid operation %1%", op); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 98b745c3a..91fc178db 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } + + std::optional queryRealisation(const DrvOutput&) override + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 467169ce8..ad1779aea 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -333,6 +333,10 @@ public: auto conn(connections->get()); return conn->remoteVersion; } + + std::optional queryRealisation(const DrvOutput&) override + // TODO: Implement + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regLegacySSHStore; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 7d979c5c2..bb7464989 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -87,6 +87,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + createDirs(binaryCacheDir + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a47b3956..418b3ab9c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -597,13 +597,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } -void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); + // XXX: This ignores the references of the output because we can + // recompute them later from the drv and the references of the associated + // store path, but doing so is both inefficient and fragile. + return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); } -void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -653,7 +656,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - linkDeriverToPath(state, id, i.first, *i.second.second); + registerDrvOutput_(state, id, i.first, *i.second.second); } } @@ -1612,5 +1615,13 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } - +std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { + auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); + if (!(outputPath && isValidPath(*outputPath))) + return std::nullopt; + else + return {DrvOutputInfo{ + .outPath = *outputPath, + }}; } +} // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 332718af4..440411f01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -208,6 +208,13 @@ public: garbage until it exceeds maxFree. */ void autoGC(bool sync = true); + /* Register the store path 'output' as the output named 'outputName' of + derivation 'deriver'. */ + void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; + void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + + std::optional queryRealisation(const DrvOutput&) override; + private: int getSchema(); @@ -276,11 +283,6 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ - void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output); - void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output); - Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc new file mode 100644 index 000000000..fcc1a3825 --- /dev/null +++ b/src/libstore/realisation.cc @@ -0,0 +1,72 @@ +#include "realisation.hh" +#include "store-api.hh" + +namespace nix { + +MakeError(InvalidDerivationOutputId, Error); + +DrvOutput DrvOutput::parse(const std::string &strRep) { + const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); + if (outputs.size() != 1) + throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); + + return DrvOutput{ + .drvPath = StorePath(rawPath), + .outputName = *outputs.begin(), + }; +} + +std::string DrvOutput::to_string() const { + return std::string(drvPath.to_string()) + "!" + outputName; +} + +std::string Realisation::to_string() const { + std::string res; + + res += "Id: " + id.to_string() + '\n'; + res += "OutPath: " + std::string(outPath.to_string()) + '\n'; + + return res; +} + +Realisation Realisation::parse(const std::string & s, const std::string & whence) +{ + // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out + auto corrupt = [&]() { + return Error("Drv output info file '%1%' is corrupt", whence); + }; + + std::optional id; + std::optional outPath; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) throw corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) throw corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "Id") + id = DrvOutput::parse(value); + + if (name == "OutPath") + outPath = StorePath(value); + + pos = eol + 1; + } + + if (!outPath) corrupt(); + if (!id) corrupt(); + return Realisation { + .id = *id, + .outPath = *outPath, + }; +} + +} // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh new file mode 100644 index 000000000..c573e1bb4 --- /dev/null +++ b/src/libstore/realisation.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct DrvOutput { + StorePath drvPath; + std::string outputName; + + std::string to_string() const; + + static DrvOutput parse(const std::string &); + + bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } + bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } + +private: + // Just to make comparison operators easier to write + std::pair to_pair() const + { return std::make_pair(drvPath, outputName); } +}; + +struct Realisation { + DrvOutput id; + StorePath outPath; + + std::string to_string() const; + static Realisation parse(const std::string & s, const std::string & whence); +}; + +typedef std::map DrvOutputs; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be29f8e6f..f1f4d0516 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } +void RemoteStore::registerDrvOutput(const Realisation & info) +{ + auto conn(getConnection()); + conn->to << wopRegisterDrvOutput; + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + conn.processStderr(); +} + +std::optional RemoteStore::queryRealisation(const DrvOutput & id) +{ + auto conn(getConnection()); + conn->to << wopQueryRealisation; + conn->to << id.to_string(); + conn.processStderr(); + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; +} + void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f78fcb02..fdd53e6ed 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -81,6 +81,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6b9331495..7cdadc1f3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "realisation.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -396,6 +397,8 @@ protected: public: + virtual std::optional queryRealisation(const DrvOutput &) = 0; + /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -468,6 +471,18 @@ public: virtual StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) = 0; + /** + * Add a mapping indicating that `deriver!outputName` maps to the output path + * `output`. + * + * This is redundant for known-input-addressed and fixed-output derivations + * as this information is already present in the drv file, but necessary for + * floating-ca derivations and their dependencies as there's no way to + * retrieve this information otherwise. + */ + virtual void registerDrvOutput(const Realisation & output) + { unsupported("registerDrvOutput"); } + /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 63bd6ea49..f2cdc7ca3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,5 +1,8 @@ #pragma once +#include "store-api.hh" +#include "serialise.hh" + namespace nix { @@ -50,6 +53,8 @@ typedef enum { wopAddToStoreNar = 39, wopQueryMissing = 40, wopQueryDerivationOutputMap = 41, + wopRegisterDrvOutput = 42, + wopQueryRealisation = 43, } WorkerOp; diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index bc37a99c1..e8ac88609 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -55,7 +55,8 @@ testNixCommand () { nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link } -testRemoteCache +# Disabled until we have it properly working +# testRemoteCache testDeterministicCA testCutoff testGC From 3ac9d74eb1de0f696bb0384132f7ecc7d057f9d6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:14:02 +0200 Subject: [PATCH 217/227] Rework the db schema for derivation outputs Add a new table for tracking the derivation output mappings. We used to hijack the `DerivationOutputs` table for that, but (despite its name), it isn't a really good fit: - Its entries depend on the drv being a valid path, making it play badly with garbage collection and preventing us to copy a drv output without copying the whole drv closure too; - It dosen't guaranty that the output path exists; By using a different table, we can experiment with a different schema better suited for tracking the output mappings of CA derivations. (incidentally, this also fixes #4138) --- src/libstore/build/derivation-goal.cc | 16 +- src/libstore/ca-specific-schema.sql | 11 ++ src/libstore/local-store.cc | 217 ++++++++++++++++++-------- src/libstore/local-store.hh | 4 +- src/libstore/local.mk | 4 +- 5 files changed, 172 insertions(+), 80 deletions(-) create mode 100644 src/libstore/ca-specific-schema.sql diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a0f10c33d..b7bf866eb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && + ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed)) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3405,12 +3406,11 @@ void DerivationGoal::registerOutputs() drvPathResolved = writeDerivation(worker.store, drv2); } - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.registerDrvOutput( - DrvOutputId{drvPathResolved, outputName}, - DrvOutputInfo{.outPath = newInfo.path, - .resolvedDrv = drvPathResolved}); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + for (auto& [outputName, newInfo] : infos) + worker.store.registerDrvOutput(Realisation{ + .id = DrvOutput{drvPathResolved, outputName}, + .outPath = newInfo.path}); } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql new file mode 100644 index 000000000..93c442826 --- /dev/null +++ b/src/libstore/ca-specific-schema.sql @@ -0,0 +1,11 @@ +-- Extension of the sql schema for content-addressed derivations. +-- Won't be loaded unless the experimental feature `ca-derivations` +-- is enabled + +create table if not exists Realisations ( + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + primary key (drvPath, outputName), + foreign key (outputPath) references ValidPaths(id) on delete cascade +); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 418b3ab9c..69ab821d9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -52,12 +52,52 @@ struct LocalStore::State::Stmts { SQLiteStmt QueryReferrers; SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; + SQLiteStmt RegisterRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryRealisedOutput; + SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; }; +int getSchema(Path schemaPath) +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error("'%1%' is corrupt", schemaPath); + } + return curSchema; +} + +void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) +{ + const int nixCASchemaVersion = 1; + int curCASchema = getSchema(schemaPath); + if (curCASchema != nixCASchemaVersion) { + if (curCASchema > nixCASchemaVersion) { + throw Error("current Nix store ca-schema is version %1%, but I only support %2%", + curCASchema, nixCASchemaVersion); + } + + if (!lockFile(lockFd.get(), ltWrite, false)) { + printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltWrite, true); + } + + if (curCASchema == 0) { + static const char schema[] = + #include "ca-specific-schema.sql.gen.hh" + ; + db.exec(schema); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + lockFile(lockFd.get(), ltRead, true); + } +} + LocalStore::LocalStore(const Params & params) : StoreConfig(params) , Store(params) @@ -238,6 +278,10 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } + /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); @@ -264,6 +308,28 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + state->stmts->RegisterRealisedOutput.create(state->db, + R"( + insert or replace into Realisations (drvPath, outputName, outputPath) + values (?, ?, (select id from ValidPaths where path = ?)) + ; + )"); + state->stmts->QueryRealisedOutput.create(state->db, + R"( + select Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? and outputName = ? + ; + )"); + state->stmts->QueryAllRealisedOutputs.create(state->db, + R"( + select outputName, Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? + ; + )"); + } } @@ -301,16 +367,7 @@ std::string LocalStore::getUri() int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error("'%1%' is corrupt", schemaPath); - } - return curSchema; -} - +{ return nix::getSchema(schemaPath); } void LocalStore::openDB(State & state, bool create) { @@ -600,13 +657,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - // XXX: This ignores the references of the output because we can - // recompute them later from the drv and the references of the associated - // store path, but doing so is both inefficient and fragile. - return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); + retrySQLite([&]() { + state->stmts->RegisterRealisedOutput.use() + (info.id.drvPath.to_string()) + (info.id.outputName) + (printStorePath(info.outPath)) + .exec(); + }); } -void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -656,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - registerDrvOutput_(state, id, i.first, *i.second.second); + cacheDrvOutputMapping(state, id, i.first, *i.second.second); } } @@ -817,70 +877,85 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } - -std::map> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +// Try to resolve the derivation at path `original`, with a caching layer +// to make it more efficient +std::optional cachedResolve( + LocalStore & store, + const StorePath & original) { - auto path = path_; - std::map> outputs; - Derivation drv = readDerivation(path); - for (auto & [outName, _] : drv.outputs) { - outputs.insert_or_assign(outName, std::nullopt); - } - bool haveCached = false; { auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(path); + auto resolvedPathOptIter = resolutions->find(original); if (resolvedPathOptIter != resolutions->end()) { auto & [_, resolvedPathOpt] = *resolvedPathOptIter; if (resolvedPathOpt) - path = *resolvedPathOpt; - haveCached = true; + return resolvedPathOpt; } } - /* can't just use else-if instead of `!haveCached` because we need to unlock - `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { - /* Try resolve drv and use that path instead. */ - auto attempt = drv.tryResolve(*this); - if (!attempt) - /* If we cannot resolve the derivation, we cannot have any path - assigned so we return the map of all std::nullopts. */ - return outputs; - /* Just compute store path */ - auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); - /* Store in memo table. */ - /* FIXME: memo logic should not be local-store specific, should have - wrapper-method instead. */ - drvPathResolutions.lock()->insert_or_assign(path, pathResolved); - path = std::move(pathResolved); - } - return retrySQLite>>([&]() { - auto state(_state.lock()); + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(original); + auto attempt = drv.tryResolve(store); + if (!attempt) + return std::nullopt; + /* Just compute store path */ + auto pathResolved = + writeDerivation(store, *std::move(attempt), NoRepair, true); + /* Store in memo table. */ + drvPathResolutions.lock()->insert_or_assign(original, pathResolved); + return pathResolved; +} + +std::map> +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +{ + auto path = path_; + auto outputs = retrySQLite>>([&]() { + auto state(_state.lock()); + std::map> outputs; uint64_t drvId; try { drvId = queryValidPathId(*state, path); - } catch (InvalidPath &) { - /* FIXME? if the derivation doesn't exist, we cannot have a mapping - for it. */ - return outputs; + } catch (InvalidPath&) { + // Ignore non-existing drvs as they might still have an output map + // defined if ca-derivations is enabled } + auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); + while (use.next()) + outputs.insert_or_assign( + use.getStr(0), parseStorePath(use.getStr(1))); - auto useQueryDerivationOutputs { - state->stmts->QueryDerivationOutputs.use() - (drvId) - }; + return outputs; + }); + + if (!settings.isExperimentalFeatureEnabled("ca-derivations")) + return outputs; + + auto drv = readDerivation(path); + + for (auto & output : drv.outputsAndOptPaths(*this)) { + outputs.emplace(output.first, std::nullopt); + } + + auto resolvedDrv = cachedResolve(*this, path); + + if (!resolvedDrv) + return outputs; + + retrySQLite([&]() { + auto state(_state.lock()); + path = *resolvedDrv; + auto useQueryDerivationOutputs{ + state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; while (useQueryDerivationOutputs.next()) outputs.insert_or_assign( useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1)) - ); - - return outputs; + parseStorePath(useQueryDerivationOutputs.getStr(1))); }); -} + return outputs; +} std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { @@ -1615,13 +1690,19 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { - auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); - if (!(outputPath && isValidPath(*outputPath))) - return std::nullopt; - else - return {DrvOutputInfo{ - .outPath = *outputPath, - }}; +std::optional LocalStore::queryRealisation( + const DrvOutput& id) { + typedef std::optional Ret; + return retrySQLite([&]() -> Ret { + auto state(_state.lock()); + auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + id.outputName)); + if (!use.next()) + return std::nullopt; + auto outputPath = parseStorePath(use.getStr(0)); + auto resolvedDrv = StorePath(use.getStr(1)); + return Ret{ + Realisation{.id = id, .outPath = outputPath}}; + }); } } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 440411f01..69559e346 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -210,8 +210,8 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ - void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; - void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + void registerDrvOutput(const Realisation & info) override; + void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); std::optional queryRealisation(const DrvOutput&) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index dfe1e2cc4..03c4351ac 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif -$(d)/local-store.cc: $(d)/schema.sql.gen.hh +$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(d)/build.cc: @@ -58,7 +58,7 @@ $(d)/build.cc: @echo ')foo"' >> $@.tmp @mv $@.tmp $@ -clean-files += $(d)/schema.sql.gen.hh +clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) From 8914e01e37ad072d940e2000fede7c2e0f4b194c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 21:07:52 +0100 Subject: [PATCH 218/227] Store the realisations as JSON in the binary cache Fix #4332 --- src/libstore/binary-cache-store.cc | 5 ++- src/libstore/realisation.cc | 61 ++++++++++-------------------- src/libstore/realisation.hh | 5 ++- 3 files changed, 25 insertions(+), 46 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 085dc7ba1..5b081c1ae 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -449,7 +449,8 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + return {Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; } else { return std::nullopt; } @@ -457,7 +458,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); + upsertFile(filePath, info.toJSON(), "application/json"); } ref BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index fcc1a3825..47db1ec9f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include namespace nix { @@ -20,52 +21,28 @@ std::string DrvOutput::to_string() const { return std::string(drvPath.to_string()) + "!" + outputName; } -std::string Realisation::to_string() const { - std::string res; - - res += "Id: " + id.to_string() + '\n'; - res += "OutPath: " + std::string(outPath.to_string()) + '\n'; - - return res; +nlohmann::json Realisation::toJSON() const { + return nlohmann::json{ + {"id", id.to_string()}, + {"outPath", outPath.to_string()}, + }; } -Realisation Realisation::parse(const std::string & s, const std::string & whence) -{ - // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out - auto corrupt = [&]() { - return Error("Drv output info file '%1%' is corrupt", whence); +Realisation Realisation::fromJSON( + const nlohmann::json& json, + const std::string& whence) { + auto getField = [&](std::string fieldName) -> std::string { + auto fieldIterator = json.find(fieldName); + if (fieldIterator == json.end()) + throw Error( + "Drv output info file '%1%' is corrupt, missing field %2%", + whence, fieldName); + return *fieldIterator; }; - std::optional id; - std::optional outPath; - - size_t pos = 0; - while (pos < s.size()) { - - size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); - - std::string name(s, pos, colon - pos); - - size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); - - std::string value(s, colon + 2, eol - colon - 2); - - if (name == "Id") - id = DrvOutput::parse(value); - - if (name == "OutPath") - outPath = StorePath(value); - - pos = eol + 1; - } - - if (!outPath) corrupt(); - if (!id) corrupt(); - return Realisation { - .id = *id, - .outPath = *outPath, + return Realisation{ + .id = DrvOutput::parse(getField("id")), + .outPath = StorePath(getField("outPath")), }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index c573e1bb4..08579b739 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,6 +1,7 @@ #pragma once #include "path.hh" +#include namespace nix { @@ -25,8 +26,8 @@ struct Realisation { DrvOutput id; StorePath outPath; - std::string to_string() const; - static Realisation parse(const std::string & s, const std::string & whence); + nlohmann::json toJSON() const; + static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); }; typedef std::map DrvOutputs; From bab1cda0e6c30e25460b5a9c809589d3948f35df Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 16:56:56 +0100 Subject: [PATCH 219/227] Use the hash modulo in the derivation outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than storing the derivation outputs as `drvPath!outputName` internally, store them as `drvHashModulo!outputName` (or `outputHash!outputName` for fixed-output derivations). This makes the storage slightly more opaque, but enables an earlier cutoff in cases where a fixed-output dependency changes (but keeps the same output hash) − same as what we already do for input-addressed derivations. --- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-goal.cc | 28 ++++-------- src/libstore/derivations.cc | 49 +++++++++++++++------ src/libstore/derivations.hh | 22 ++++------ src/libstore/local-store.cc | 61 ++++++++++++--------------- src/libstore/realisation.cc | 10 ++--- src/libstore/realisation.hh | 10 +++-- 7 files changed, 93 insertions(+), 89 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..d059e3daf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1107,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // Shouldn't happen as the toplevel derivation is not CA. assert(false); }, - [&](UnknownHashes) { + [&](DeferredHash _) { for (auto & i : outputs) { drv.outputs.insert_or_assign(i, DerivationOutput { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b7bf866eb..54b37553a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -504,9 +504,6 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -2097,15 +2094,15 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf void registerDrvOutput(const Realisation & info) override { - if (!goal.isAllowed(info.id.drvPath)) - throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how next->registerDrvOutput(info); } std::optional queryRealisation(const DrvOutput & id) override { - if (!goal.isAllowed(id.drvPath)) - throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how return next->queryRealisation(id); } @@ -3394,23 +3391,14 @@ void DerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (settings.isExperimentalFeatureEnabled("ca-derivations")) + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto outputHashes = staticOutputHashes(worker.store, *drv); for (auto& [outputName, newInfo] : infos) worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{drvPathResolved, outputName}, + .id = DrvOutput{outputHashes.at(outputName), outputName}, .outPath = newInfo.path}); + } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 231ca26c2..5bcc7f012 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { + bool isDeferred = false; /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { - case DerivationType::CAFloating: - return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } + case DerivationType::CAFloating: + isDeferred = true; + break; case DerivationType::InputAddressed: break; case DerivationType::DeferredInputAddressed: @@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { - bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation [&](Hash drvHash) { inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); }, + [&](DeferredHash deferredHash) { + isDeferred = true; + inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); + }, // CA derivation's output hashes [&](CaOutputHashes outputHashes) { std::set justOut = { "out" }; @@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, - [&](UnknownHashes) { - hasUnknownHash = true; - }, }, res); - if (hasUnknownHash) { - return UnknownHashes {}; - } } - return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + + if (isDeferred) + return DeferredHash { hash }; + else + return hash; +} + + +std::map staticOutputHashes(Store& store, const Derivation& drv) +{ + std::map res; + std::visit(overloaded { + [&](Hash drvHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, drvHash}); + } + }, + [&](DeferredHash deferredHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, deferredHash.hash}); + } + }, + [&](CaOutputHashes outputHashes) { + res = outputHashes; + }, + }, hashDerivationModulo(store, drv, true)); + return res; } @@ -719,9 +745,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } - -Sync drvPathResolutions; - std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b966d6d90..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -18,8 +18,6 @@ namespace nix { /* The traditional non-fixed-output derivation type. */ struct DerivationOutputInputAddressed { - /* Will need to become `std::optional` once input-addressed - derivations are allowed to depend on cont-addressed derivations */ StorePath path; }; @@ -174,12 +172,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; -struct UnknownHashes {}; +struct DeferredHash { Hash hash; }; typedef std::variant< Hash, // regular DRV normalized hash CaOutputHashes, // Fixed-output derivation hashes - UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies + DeferredHash // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations @@ -207,22 +205,18 @@ typedef std::variant< */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +/* + Return a map associating each output to a hash that uniquely identifies its + derivation (modulo the self-references). + */ +std::map staticOutputHashes(Store& store, const Derivation& drv); + /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; -/* Memoisation of `readDerivation(..).resove()`. */ -typedef std::map< - StorePath, - std::optional -> DrvPathResolutions; - -// FIXME: global, though at least thread-safe. -// FIXME: arguably overlaps with hashDerivationModulo memo table. -extern Sync drvPathResolutions; - bool wantOutput(const string & output, const std::set & wanted); struct Source; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 69ab821d9..1539c94e2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -659,7 +659,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() - (info.id.drvPath.to_string()) + (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) .exec(); @@ -879,17 +879,18 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) // Try to resolve the derivation at path `original`, with a caching layer // to make it more efficient -std::optional cachedResolve( - LocalStore & store, - const StorePath & original) +std::optional cachedResolve( + LocalStore& store, + const StorePath& original) { + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; { - auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(original); - if (resolvedPathOptIter != resolutions->end()) { - auto & [_, resolvedPathOpt] = *resolvedPathOptIter; - if (resolvedPathOpt) - return resolvedPathOpt; + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(original); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; } } @@ -898,12 +899,9 @@ std::optional cachedResolve( auto attempt = drv.tryResolve(store); if (!attempt) return std::nullopt; - /* Just compute store path */ - auto pathResolved = - writeDerivation(store, *std::move(attempt), NoRepair, true); /* Store in memo table. */ - drvPathResolutions.lock()->insert_or_assign(original, pathResolved); - return pathResolved; + resolutionsCache.lock()->insert_or_assign(original, *attempt); + return *attempt; } std::map> @@ -933,26 +931,24 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto drv = readDerivation(path); - for (auto & output : drv.outputsAndOptPaths(*this)) { - outputs.emplace(output.first, std::nullopt); - } - auto resolvedDrv = cachedResolve(*this, path); - if (!resolvedDrv) + if (!resolvedDrv) { + for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { + if (!outputs.count(outputName)) + outputs.emplace(outputName, std::nullopt); + } return outputs; + } - retrySQLite([&]() { - auto state(_state.lock()); - path = *resolvedDrv; - auto useQueryDerivationOutputs{ - state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; - - while (useQueryDerivationOutputs.next()) - outputs.insert_or_assign( - useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1))); - }); + auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); + for (auto& [outputName, hash] : resolvedDrvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) + outputs.insert_or_assign(outputName, realisation->outPath); + else + outputs.insert_or_assign(outputName, std::nullopt); + } return outputs; } @@ -1695,12 +1691,11 @@ std::optional LocalStore::queryRealisation( typedef std::optional Ret; return retrySQLite([&]() -> Ret { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( id.outputName)); if (!use.next()) return std::nullopt; auto outputPath = parseStorePath(use.getStr(0)); - auto resolvedDrv = StorePath(use.getStr(1)); return Ret{ Realisation{.id = id, .outPath = outputPath}}; }); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47db1ec9f..47ad90eee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -7,18 +7,18 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); DrvOutput DrvOutput::parse(const std::string &strRep) { - const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); - if (outputs.size() != 1) + size_t n = strRep.find("!"); + if (n == strRep.npos) throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); return DrvOutput{ - .drvPath = StorePath(rawPath), - .outputName = *outputs.begin(), + .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), + .outputName = strRep.substr(n+1), }; } std::string DrvOutput::to_string() const { - return std::string(drvPath.to_string()) + "!" + outputName; + return strHash() + "!" + outputName; } nlohmann::json Realisation::toJSON() const { diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 08579b739..4b8ead3c5 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -6,11 +6,15 @@ namespace nix { struct DrvOutput { - StorePath drvPath; + // The hash modulo of the derivation + Hash drvHash; std::string outputName; std::string to_string() const; + std::string strHash() const + { return drvHash.to_string(Base16, true); } + static DrvOutput parse(const std::string &); bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } @@ -18,8 +22,8 @@ struct DrvOutput { private: // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvPath, outputName); } + std::pair to_pair() const + { return std::make_pair(drvHash, outputName); } }; struct Realisation { From e9b39f6004ec68f062230514534b08033cf133c7 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Dec 2020 21:12:53 +0100 Subject: [PATCH 220/227] Restrict the operations on drv outputs in recursive Nix There's currently no way to properly filter them, so disallow them altogether instead. --- src/libstore/build/derivation-goal.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 54b37553a..f494545fb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2093,18 +2093,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf } void registerDrvOutput(const Realisation & info) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - next->registerDrvOutput(info); - } + // XXX: This should probably be allowed as a no-op if the realisation + // corresponds to an allowed derivation + { throw Error("registerDrvOutput"); } std::optional queryRealisation(const DrvOutput & id) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - return next->queryRealisation(id); - } + // XXX: This should probably be allowed if the realisation corresponds to + // an allowed derivation + { throw Error("queryRealisation"); } void buildPaths(const std::vector & paths, BuildMode buildMode) override { From f890830b337f321f17ad36b45d7d63801a753554 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 11 Dec 2020 16:31:14 +0100 Subject: [PATCH 221/227] primops/fromJSON: add error position in case of parse error This makes it easier to track down where invalid JSON was passed to `builtins.fromJSON`. --- src/libexpr/primops.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..63624c520 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1621,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) { string s = state.forceStringNoCtx(*args[0], pos); - parseJSON(state, s, v); + try { + parseJSON(state, s, v); + } catch (JSONParseError &e) { + e.addTrace(pos, "while decoding a JSON string"); + throw e; + } } static RegisterPrimOp primop_fromJSON({ From 44c3fbc6e03ec518f6174c2b7c21b603973beb91 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:03:54 +0200 Subject: [PATCH 222/227] Fix `addTextToStore` for binary caches Because of a too eager refactoring, `addTextToStore` used to throw an error because the input wasn't a valid nar. Partially revert that refactoring to wrap the text into a proper nar (using `dumpString`) to make this method work again --- src/libstore/binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5b081c1ae..94c11355f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s if (!repair && isValidPath(path)) return path; - auto source = StringSource { s }; + StringSink sink; + dumpString(s, sink); + auto source = StringSource { *sink.s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { path, nar.first }; info.narSize = nar.second; From 6e899278d305da904fb766937f56344841c022b3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 6 Nov 2020 09:51:17 +0100 Subject: [PATCH 223/227] Better detect when `buildPaths` would be a no-op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildPaths` can be called even for stores where it's not defined in case it's bound to be a no-op. The “no-op detection” mechanism was only detecting the case wher `buildPaths` was called on a set of (non-drv) paths that were already present on the store. This commit extends this mechanism to also detect the case where `buildPaths` is called on a set of derivation outputs which are already built on the store. This only works with the ca-derivations flag. It could be possible to extend this to also work without it, but it would add quite a bit of complexity, and it's not used without it anyways. --- src/libstore/store-api.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..50905bb33 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -729,9 +729,17 @@ void Store::buildPaths(const std::vector & paths, BuildMod StorePathSet paths2; for (auto & path : paths) { - if (path.path.isDerivation()) - unsupported("buildPaths"); - paths2.insert(path.path); + if (path.path.isDerivation()) { + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & outputName : path.outputs) { + if (!queryRealisation({path.path, outputName})) + unsupported("buildPaths"); + } + } else + unsupported("buildPaths"); + + } else + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) From 962b82ef25069893779ed56d31e44814793f9273 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:34:45 +0100 Subject: [PATCH 224/227] Fix BinaryCacheStore::registerDrvOutput Was crashing because coercing a json document into a string is only valid if the json is a string, otherwise we need to call `.dump()` --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 94c11355f..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -460,7 +460,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.toJSON(), "application/json"); + upsertFile(filePath, info.toJSON().dump(), "application/json"); } ref BinaryCacheStore::getFSAccessor() From cac8d5b742ec0cb80ad7232e20f63c74a217e545 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 16 Dec 2020 13:36:17 +0100 Subject: [PATCH 225/227] Don't ignore an absent drv file in queryPartialDrvOutputMap This ignore was here because `queryPartialDrvOutputMap` was used both 1. as a cache to avoid having to re-read the derivation (when gc-ing for example), and 2. as the source of truth for ca realisations The use-case 2. required it to be able to work even when the derivation wasn't there anymore (see https://github.com/NixOS/nix/issues/4138). However, this use-case is now handled by `queryRealisation`, meaning that we can safely error out if the derivation isn't there anymore --- src/libstore/local-store.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1539c94e2..20bbc73cf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -912,12 +912,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto state(_state.lock()); std::map> outputs; uint64_t drvId; - try { - drvId = queryValidPathId(*state, path); - } catch (InvalidPath&) { - // Ignore non-existing drvs as they might still have an output map - // defined if ca-derivations is enabled - } + drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); while (use.next()) outputs.insert_or_assign( From 4d458394991f3086c3c9c306d000e6c0058c4fa7 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 17 Dec 2020 11:35:24 +0100 Subject: [PATCH 226/227] Fix the detection of already built drv outputs PRs #4370 and #4348 had a bad interaction in that the second broke the fist one in a not trivial way. The issue was that since #4348 the logic for detecting whether a derivation output is already built requires some logic that was specific to the `LocalStore`. It happens though that most of this logic could be upstreamed to any `Store`, which is what this commit does. --- src/libstore/derivations.cc | 32 +++++++++++++++++++++++++- src/libstore/derivations.hh | 4 ++++ src/libstore/local-store.cc | 45 ++++--------------------------------- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 39 +++++++++++++++++++++++++------- src/libstore/store-api.hh | 9 ++++++-- 6 files changed, 78 insertions(+), 53 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5bcc7f012..7466c7d41 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) { +std::optional Derivation::tryResolveUncached(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -771,4 +771,34 @@ std::optional Derivation::tryResolve(Store & store) { return resolved; } +std::optional Derivation::tryResolve(Store& store) +{ + auto drvPath = writeDerivation(store, *this, NoRepair, false); + return Derivation::tryResolve(store, drvPath); +} + +std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) +{ + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; + + { + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(drvPath); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; + } + } + + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(drvPath); + auto attempt = drv.tryResolveUncached(store); + if (!attempt) + return std::nullopt; + /* Store in memo table. */ + resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); + return *attempt; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4e5985fab..3d8f19aef 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,10 +138,14 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); + static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + +private: + std::optional tryResolveUncached(Store & store); }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 20bbc73cf..e9f9bde4d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -877,35 +877,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } -// Try to resolve the derivation at path `original`, with a caching layer -// to make it more efficient -std::optional cachedResolve( - LocalStore& store, - const StorePath& original) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(original); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(original); - auto attempt = drv.tryResolve(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(original, *attempt); - return *attempt; -} std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { @@ -924,20 +898,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) if (!settings.isExperimentalFeatureEnabled("ca-derivations")) return outputs; - auto drv = readDerivation(path); - - auto resolvedDrv = cachedResolve(*this, path); - - if (!resolvedDrv) { - for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { - if (!outputs.count(outputName)) - outputs.emplace(outputName, std::nullopt); - } - return outputs; - } - - auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); - for (auto& [outputName, hash] : resolvedDrvHashes) { + auto drv = readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto& [outputName, hash] : drvHashes) { auto realisation = queryRealisation(DrvOutput{hash, outputName}); if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69559e346..877dba742 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 50905bb33..2cd39ab11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +{ + std::map> outputs; + auto drv = readInvalidDerivation(path); + for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + outputs.emplace(outputName, output.second); + } + return outputs; +} + +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +{ + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = Derivation::tryResolve(*this, path); + if (resolvedDrv) { + auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); + if (isValidPath(resolvedDrvPath)) + return queryDerivationOutputMapNoResolve(resolvedDrvPath); + } + } + return queryDerivationOutputMapNoResolve(path); +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; @@ -730,14 +753,14 @@ void Store::buildPaths(const std::vector & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto & outputName : path.outputs) { - if (!queryRealisation({path.path, outputName})) - unsupported("buildPaths"); - } - } else - unsupported("buildPaths"); - + auto outPaths = queryPartialDerivationOutputMap(path.path); + for (auto & outputName : path.outputs) { + auto currentOutputPathIter = outPaths.find(outputName); + if (currentOutputPathIter == outPaths.end() || + !currentOutputPathIter->second || + !isValidPath(*currentOutputPathIter->second)) + unsupported("buildPaths"); + } } else paths2.insert(path.path); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7cdadc1f3..ce95b78b1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -416,8 +416,13 @@ public: /* Query the mapping outputName => outputPath for the given derivation. All outputs are mentioned so ones mising the mapping are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path) - { unsupported("queryPartialDerivationOutputMap"); } + virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + + /* + * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve + * the derivation + */ + virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ From 7de4b1e9aa72c21f5c98001ea2a5f312d26634b7 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Thu, 17 Dec 2020 23:42:49 +0100 Subject: [PATCH 227/227] smaller fixes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/command-ref/nix.md | 1 - doc/manual/src/contributing/hacking.md | 76 +++++++++++++++++++++++++ doc/manual/src/hacking.md | 77 -------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) delete mode 100644 doc/manual/src/command-ref/nix.md delete mode 100644 doc/manual/src/hacking.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index fdb1f7969..448fee803 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -62,6 +62,7 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) +@manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md deleted file mode 100644 index cc677d127..000000000 --- a/doc/manual/src/command-ref/nix.md +++ /dev/null @@ -1 +0,0 @@ -# nix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2ad773dea..2a1e55e5b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -1 +1,77 @@ # Hacking + +This section provides some notes on how to hack on Nix. To get the +latest version of Nix from GitHub: + +```console +$ git clone https://github.com/NixOS/nix.git +$ cd nix +``` + +To build Nix for the current operating system/architecture use + +```console +$ nix-build +``` + +or if you have a flake-enabled nix: + +```console +$ nix build +``` + +This will build `defaultPackage` attribute defined in the `flake.nix` +file. To build for other platforms add one of the following suffixes to +it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. + +```console +$ nix-build -A defaultPackage.x86_64-linux +``` + +To build all dependencies and start a shell in which all environment +variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To build Nix itself in this shell: + +```console +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 3.0 +``` + +To run a functional test: + +```console +make tests/test-name-should-auto-complete.sh.test +``` + +To run the unit-tests for C++ code: + +``` +make check +``` + +If you have a flakes-enabled Nix you can replace: + +```console +$ nix-shell +``` + +by: + +```console +$ nix develop +``` diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md deleted file mode 100644 index 2a1e55e5b..000000000 --- a/doc/manual/src/hacking.md +++ /dev/null @@ -1,77 +0,0 @@ -# Hacking - -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -To build Nix for the current operating system/architecture use - -```console -$ nix-build -``` - -or if you have a flake-enabled nix: - -```console -$ nix build -``` - -This will build `defaultPackage` attribute defined in the `flake.nix` -file. To build for other platforms add one of the following suffixes to -it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. - -```console -$ nix-build -A defaultPackage.x86_64-linux -``` - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To build Nix itself in this shell: - -```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 3.0 -``` - -To run a functional test: - -```console -make tests/test-name-should-auto-complete.sh.test -``` - -To run the unit-tests for C++ code: - -``` -make check -``` - -If you have a flakes-enabled Nix you can replace: - -```console -$ nix-shell -``` - -by: - -```console -$ nix develop -```