From 88cf6ffce3c01f4f1c50250ef46c0d7bf23f41c7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 16:27:00 -0400 Subject: [PATCH 001/477] Rename logging->stdout to logging->stdout_ musl doesn't like this identifier --- src/libutil/logging.hh | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/hash.cc | 4 ++-- src/nix/ls.cc | 4 ++-- src/nix/show-config.cc | 2 +- src/nix/why-depends.cc | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index b1583eced..46deb89f7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -87,7 +87,7 @@ public: virtual void writeToStdout(std::string_view s); template - inline void stdout(const std::string & fs, const Args & ... args) + inline void stdout_(const std::string & fs, const Args & ... args) { boost::format f(fs); formatHelper(f, args...); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d6de16e..745c24748 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -58,7 +58,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand store->addToStore(info, source); } - logger->stdout("%s", store->printStorePath(info.path)); + logger->stdout_("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 26e98ac2a..53ec8c920 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -65,7 +65,7 @@ struct CmdEval : MixJSON, InstallableCommand printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - logger->stdout("%s", *v); + logger->stdout_("%s", *v); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..cdc8bf767 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -69,7 +69,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->stdout(h.to_string(base, base == SRI)); + logger->stdout_(h.to_string(base, base == SRI)); } } }; @@ -103,7 +103,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout(Hash(s, ht).to_string(base, base == SRI)); + logger->stdout_(Hash(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d2157f2d4..59922a8de 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); - logger->stdout(line); + logger->stdout_(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - logger->stdout(relPath); + logger->stdout_(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 4fd8886de..a97dc42f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -25,7 +25,7 @@ struct CmdShowConfig : Command, MixJSON std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - logger->stdout("%s = %s", s.first, s.second.value); + logger->stdout_("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 167c974ee..5e4d5fdcf 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -152,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - logger->stdout("%s%s%s%s" ANSI_NORMAL, + logger->stdout_("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From 07dae2ff7727b915a1b687cdec9aca894d7c2f72 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:19:11 -0400 Subject: [PATCH 002/477] Setup static building of nix --- release-common.nix | 37 ++++++++------ release.nix | 117 ++++++++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/release-common.nix b/release-common.nix index 4316c3c23..2cf9c233e 100644 --- a/release-common.nix +++ b/release-common.nix @@ -1,4 +1,4 @@ -{ pkgs }: +{ pkgs, enableStatic }: with pkgs; @@ -30,35 +30,42 @@ rec { }); configureFlags = - lib.optionals stdenv.isLinux [ + lib.optionals (!enableStatic && stdenv.isLinux) [ "--with-sandbox-shell=${sh}/bin/busybox" ]; + nativeBuildDeps = + [ + buildPackages.bison + buildPackages.flex + buildPackages.libxml2 + buildPackages.libxslt + buildPackages.docbook5 + buildPackages.docbook_xsl_ns + buildPackages.autoreconfHook + buildPackages.pkgconfig + + # Tests + buildPackages.git + buildPackages.mercurial + buildPackages.ipfs + ]; + buildDeps = - [ bison - flex - libxml2 - libxslt - docbook5 - docbook_xsl_ns + [ autoreconfHook autoconf-archive - autoreconfHook curl bzip2 xz brotli zlib editline - openssl pkgconfig sqlite + openssl sqlite libarchive boost nlohmann_json - - # Tests - git - mercurial gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + ++ lib.optional (!enableStatic && (stdenv.isLinux || stdenv.isDarwin)) ((aws-sdk-cpp.override { apis = ["s3" "transfer"]; customMemoryManagement = false; diff --git a/release.nix b/release.nix index fbf9e4721..d2785be13 100644 --- a/release.nix +++ b/release.nix @@ -12,63 +12,72 @@ let builtins.readFile ./.version + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); + buildFun = pkgs: enableStatic: + with pkgs; with import ./release-common.nix { inherit pkgs enableStatic; }; + stdenv.mkDerivation { + name = "nix-${version}"; + + src = nix; + + outputs = [ "out" "dev" "doc" ]; + + buildInputs = buildDeps; + + nativeBuildInputs = nativeBuildDeps; + + propagatedBuildInputs = propagatedDeps; + + preConfigure = + lib.optionalString (!enableStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + + (cd perl; autoreconf --install --force --verbose) + ''; + + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + dontUpdateAutotoolsGnuConfigScripts = true; + + enableParallelBuilding = true; + + makeFlags = [ "profiledir=$(out)/etc/profile.d" "PRECOMPILE_HEADERS=0" ]; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ''; + + doCheck = true; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + + separateDebugInfo = !enableStatic; + + stripAllList = ["bin"]; + }; + + jobs = rec { + + build-static = pkgs.lib.genAttrs systems (system: + buildFun (import nixpkgs { inherit system; }).pkgsStatic true); + + build = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - stdenv.mkDerivation { - name = "nix-${version}"; - - src = nix; - - outputs = [ "out" "dev" "doc" ]; - - buildInputs = buildDeps; - - propagatedBuildInputs = propagatedDeps; - - preConfigure = - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - - (cd perl; autoreconf --install --force --verbose) - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ''; - - doCheck = true; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - - separateDebugInfo = true; - }); + buildFun (import nixpkgs { inherit system; }) false); perlBindings = pkgs.lib.genAttrs systems (system: From 70719a9dd8c3d91e1d6a83d4ec9a48023cddaecf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:20:06 -0400 Subject: [PATCH 003/477] Add -lz to end of linking this is needed for static linking to work properly --- mk/programs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/programs.mk b/mk/programs.mk index 3fa9685c3..a96ff56af 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz $(1)_INSTALL_DIR ?= $$(bindir) From da77331cb740ad7d5f39dcf6d64025610ec40555 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:20:29 -0400 Subject: [PATCH 004/477] Remove lazy lookup in getHome this seems to break in Musl/Static with: terminate called after throwing an instance of 'std::bad_function_call' what(): bad_function_call --- src/libutil/lazy.hh | 48 --------------------------------------------- src/libutil/util.cc | 6 +++--- 2 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 src/libutil/lazy.hh diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh deleted file mode 100644 index d073e486c..000000000 --- a/src/libutil/lazy.hh +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include - -namespace nix { - -/* A helper class for lazily-initialized variables. - - Lazy var([]() { return value; }); - - declares a variable of type T that is initialized to 'value' (in a - thread-safe way) on first use, that is, when var() is first - called. If the initialiser code throws an exception, then all - subsequent calls to var() will rethrow that exception. */ -template -class Lazy -{ - - typedef std::function Init; - - Init init; - - std::once_flag done; - - T value; - - std::exception_ptr ex; - -public: - - Lazy(Init init) : init(init) - { } - - const T & operator () () - { - std::call_once(done, [&]() { - try { - value = init(); - } catch (...) { - ex = std::current_exception(); - } - }); - if (ex) std::rethrow_exception(ex); - return value; - } -}; - -} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1268b146a..ebb1383f3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,3 @@ -#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -511,7 +510,8 @@ std::string getUserName() } -static Lazy getHome2([]() { +static Path getHome2() +{ auto homeDir = getEnv("HOME"); if (!homeDir) { std::vector buf(16384); @@ -523,7 +523,7 @@ static Lazy getHome2([]() { homeDir = pw->pw_dir; } return *homeDir; -}); +}; Path getHome() { return getHome2(); } From 289558dffbccb47191e79629e955009b10b9888e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:39:35 -0400 Subject: [PATCH 005/477] Add unordered_set to globals.cc header --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index bee94cbd8..fa8799314 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -8,7 +8,7 @@ #include #include #include - +#include namespace nix { From 78fadaf863536d23d45b2c480139a9f77d579f9e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 17:44:57 -0400 Subject: [PATCH 006/477] fix release.nix eval --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index d2785be13..863db900b 100644 --- a/release.nix +++ b/release.nix @@ -191,7 +191,7 @@ let coverage = with pkgs; - with import ./release-common.nix { inherit pkgs; }; + with import ./release-common.nix { inherit pkgs; enableStatic = false; }; releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; From ded65899538f6a4628e711abbbdf27ea47772742 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jun 2020 18:04:16 -0400 Subject: [PATCH 007/477] Fixup coverage build --- release.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/release.nix b/release.nix index 863db900b..60cbca17b 100644 --- a/release.nix +++ b/release.nix @@ -200,6 +200,7 @@ let enableParallelBuilding = true; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps ++ propagatedDeps; dontInstall = false; From 24da034bc3ae8514ae19dadfecf6038452a5290a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 21:21:27 +0000 Subject: [PATCH 008/477] Add possibly missing `` include --- src/libutil/types.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0..2170e4c93 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include From 696bb134c1c5882cf258e3c8a480b40239cb1a9a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 21:36:09 +0000 Subject: [PATCH 009/477] Fix shell.nix --- shell.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell.nix b/shell.nix index 17aaa05ed..1addc06be 100644 --- a/shell.nix +++ b/shell.nix @@ -1,8 +1,8 @@ -{ useClang ? false }: +{ useClang ? false, enableStatic ? false }: with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {}; -with import ./release-common.nix { inherit pkgs; }; +with import ./release-common.nix { inherit pkgs enableStatic; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; From baaab2aab58aa3c47517d4ba9121a29a7ad73078 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jun 2020 14:53:40 +0000 Subject: [PATCH 010/477] Add `nativeBuildInputs` to shell.nix --- shell.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell.nix b/shell.nix index 1addc06be..75bb6ac1b 100644 --- a/shell.nix +++ b/shell.nix @@ -7,6 +7,8 @@ with import ./release-common.nix { inherit pkgs enableStatic; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ propagatedDeps ++ perlDeps; inherit configureFlags; From 13ef7a07b9be1dff894e8156a117ee3248241874 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:49:45 -0500 Subject: [PATCH 011/477] Fix build --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index bf2066157..40d779355 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,7 @@ # Tests buildPackages.git buildPackages.mercurial + buildPackages.jq ]; buildDeps = From 3537670fefab6c65b4b87837112d64931dcda4cf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:49:52 -0500 Subject: [PATCH 012/477] Only enable static on linux --- flake.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 40d779355..20c5089ee 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,8 @@ officialRelease = false; - systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; + systems = linuxSystems ++ [ "x86_64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); @@ -209,7 +210,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - build-static = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix-static); + build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -423,13 +424,14 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - build-static = self.hydraJobs.build-static.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { + build-static = self.hydraJobs.build-static.${system}; }); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; - + } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { From e12bcabdcbddc228d7af157bb3c2090e324c59a7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 02:30:12 +0000 Subject: [PATCH 013/477] Remove duplicate buildInputs --- flake.nix | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index 20c5089ee..843f9a85f 100644 --- a/flake.nix +++ b/flake.nix @@ -69,6 +69,7 @@ buildPackages.libxslt buildPackages.docbook5 buildPackages.docbook_xsl_ns + buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkgconfig @@ -79,18 +80,9 @@ ]; buildDeps = - [ bison - flex - libxml2 - libxslt - docbook5 - docbook_xsl_ns - autoconf-archive - autoreconfHook - - curl + [ curl bzip2 xz brotli zlib editline - openssl pkgconfig sqlite + openssl sqlite libarchive boost (if lib.versionAtLeast lib.version "20.03pre" @@ -178,14 +170,17 @@ src = self; + nativeBuildInputs = + [ buildPackages.autoconf-archive + buildPackages.autoreconfHook + buildPackages.pkgconfig + ]; + buildInputs = - [ autoconf-archive - autoreconfHook - nix + [ nix curl bzip2 xz - pkgconfig pkgs.perl boost ] From ec14465a001387f8972c1b8332293d4fbce5ec97 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 02:43:56 +0000 Subject: [PATCH 014/477] Separate lowdown lib and bin to be more precise --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 82eea55e2..9c79c0bbf 100644 --- a/flake.nix +++ b/flake.nix @@ -66,7 +66,7 @@ [ buildPackages.bison buildPackages.flex - buildPackages.lowdown + (lib.getBin buildPackages.lowdown) buildPackages.mdbook buildPackages.autoconf-archive buildPackages.autoreconfHook @@ -208,7 +208,7 @@ src = lowdown-src; - outputs = [ "out" "dev" ]; + outputs = [ "out" "bin" "dev" ]; nativeBuildInputs = [ which ]; @@ -216,7 +216,7 @@ '' ./configure \ PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "out"}/bin + BINDIR=${placeholder "bin"}/bin ''; }; From cfe791a638a3fdf53a2608f885c407bafc238094 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Sep 2020 11:30:04 -0400 Subject: [PATCH 015/477] stdout_ -> cout Better to get creative than just sprinkle arbitrary underscores. --- src/libutil/logging.hh | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 36 ++++++++++++++++++------------------ src/nix/hash.cc | 4 ++-- src/nix/ls.cc | 4 ++-- src/nix/profile.cc | 2 +- src/nix/registry.cc | 2 +- src/nix/search.cc | 6 +++--- src/nix/show-config.cc | 4 ++-- src/nix/why-depends.cc | 2 +- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 77b92fb51..e3fe613e8 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -100,7 +100,7 @@ public: virtual void writeToStdout(std::string_view s); template - inline void stdout_(const std::string & fs, const Args & ... args) + inline void cout(const std::string & fs, const Args & ... args) { boost::format f(fs); formatHelper(f, args...); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index df55d1bc4..04ab664b3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -83,7 +83,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand store->addToStore(info, source); } - logger->stdout_("%s", store->printStorePath(info.path)); + logger->cout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 9689f2bdb..754ffc911 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableCommand printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - logger->stdout_("%s", *v); + logger->cout("%s", *v); } } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 64fc896d9..90ffaad7c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -62,17 +62,17 @@ public: static void printFlakeInfo(const Store & store, const Flake & flake) { - logger->stdout_("Resolved URL: %s", flake.resolvedRef.to_string()); - logger->stdout_("Locked URL: %s", flake.lockedRef.to_string()); + logger->cout("Resolved URL: %s", flake.resolvedRef.to_string()); + logger->cout("Locked URL: %s", flake.lockedRef.to_string()); if (flake.description) - logger->stdout_("Description: %s", *flake.description); - logger->stdout_("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); + logger->cout("Description: %s", *flake.description); + logger->cout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); if (auto rev = flake.lockedRef.input.getRev()) - logger->stdout_("Revision: %s", rev->to_string(Base16, false)); + logger->cout("Revision: %s", rev->to_string(Base16, false)); if (auto revCount = flake.lockedRef.input.getRevCount()) - logger->stdout_("Revisions: %s", *revCount); + logger->cout("Revisions: %s", *revCount); if (auto lastModified = flake.lockedRef.input.getLastModified()) - logger->stdout_("Last modified: %s", + logger->cout("Last modified: %s", std::put_time(std::localtime(&*lastModified), "%F %T")); } @@ -140,7 +140,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON if (json) { auto json = flakeToJson(*store, flake); - logger->stdout_("%s", json.dump()); + logger->cout("%s", json.dump()); } else printFlakeInfo(*store, flake); } @@ -158,9 +158,9 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON auto flake = lockFlake(); if (json) - logger->stdout_("%s", flake.lockFile.toJson()); + logger->cout("%s", flake.lockFile.toJson()); else { - logger->stdout_("%s", flake.flake.lockedRef); + logger->cout("%s", flake.flake.lockedRef); std::unordered_set> visited; @@ -172,7 +172,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); @@ -180,7 +180,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); } else if (auto follows = std::get_if<1>(&input.second)) { - logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), input.first, printInputPath(*follows)); } @@ -811,7 +811,7 @@ struct CmdFlakeShow : FlakeCommand try { auto recurse = [&]() { - logger->stdout_("%s", headerPrefix); + logger->cout("%s", headerPrefix); auto attrs = visitor.getAttrs(); for (const auto & [i, attr] : enumerate(attrs)) { bool last = i + 1 == attrs.size(); @@ -837,7 +837,7 @@ struct CmdFlakeShow : FlakeCommand } */ - logger->stdout_("%s: %s '%s'", + logger->cout("%s: %s '%s'", headerPrefix, attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : @@ -885,7 +885,7 @@ struct CmdFlakeShow : FlakeCommand if (attrPath.size() == 1) recurse(); else if (!showLegacy) - logger->stdout_("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); + logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); else { if (visitor.isDerivation()) showDerivation(); @@ -902,7 +902,7 @@ struct CmdFlakeShow : FlakeCommand auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") throw EvalError("not an app definition"); - logger->stdout_("%s: app", headerPrefix); + logger->cout("%s: app", headerPrefix); } else if ( @@ -910,11 +910,11 @@ struct CmdFlakeShow : FlakeCommand (attrPath.size() == 2 && attrPath[0] == "templates")) { auto description = visitor.getAttr("description")->getString(); - logger->stdout_("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); } else { - logger->stdout_("%s: %s", + logger->cout("%s: %s", headerPrefix, attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 52e41e50c..945d8e990 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -73,7 +73,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->stdout_(h.to_string(base, base == SRI)); + logger->cout(h.to_string(base, base == SRI)); } } }; @@ -107,7 +107,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout_(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index b1cf92692..a9664e7c3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); - logger->stdout_(line); + logger->cout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - logger->stdout_(relPath); + logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f97df4d9e..def7db03b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -389,7 +389,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - logger->stdout_("%d %s %s %s", i, + logger->cout("%d %s %s %s", i, element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index afa3503b8..941785e55 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -26,7 +26,7 @@ struct CmdRegistryList : StoreCommand for (auto & registry : registries) { for (auto & entry : registry->entries) { // FIXME: format nicely - logger->stdout_("%s %s %s", + logger->cout("%s %s %s", registry->type == Registry::Flag ? "flags " : registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : diff --git a/src/nix/search.cc b/src/nix/search.cc index 88815efdb..2f7eb23bb 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -147,13 +147,13 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("description", description); } else { auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); - if (results > 1) logger->stdout_(""); - logger->stdout_( + if (results > 1) logger->cout(""); + logger->cout( "* %s%s", wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") - logger->stdout_( + logger->cout( " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); } } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 01a49f107..328cd2ff2 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -20,12 +20,12 @@ struct CmdShowConfig : Command, MixJSON { if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). - logger->stdout_("%s", globalConfig.toJSON().dump()); + logger->cout("%s", globalConfig.toJSON().dump()); } else { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - logger->stdout_("%s = %s", s.first, s.second.value); + logger->cout("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index cbfc9b948..f49d19ab2 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -156,7 +156,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - logger->stdout_("%s%s%s%s" ANSI_NORMAL, + logger->cout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From bd5328814fe8055b3f832a087afcf3ef11b06372 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Sat, 26 Sep 2020 14:32:58 -0700 Subject: [PATCH 016/477] Add some internal documentation for flake support objects. --- src/libexpr/flake/flake.hh | 29 +++++++++++++++++++++++++---- src/libexpr/flake/flakeref.hh | 15 +++++++++++++++ src/libfetchers/fetchers.hh | 16 ++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 69c779af8..40476a137 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,20 +17,41 @@ struct FlakeInput; typedef std::map FlakeInputs; +// FlakeInput is the flake-level parsed form of the "input" entries in +// the flake file. +// +// A FlakeInput is normally constructed by initially +// first constructing a FlakeRef (a fetcher, the fetcher-specific +// representation of the input specification, and the fetched local +// store path result) and then creating this FlakeInput to hold that +// FlakeRef, along with anything that might override that FlakeRef +// (like command-line overrides or "follows" specifications). +// +// A FlakeInput is also sometimes constructed directly from a FlakeRef +// instead of starting at the flake-file input specification +// (e.g. overrides, follows, and implicit inputs). +// +// A FlakeInput will usually have one of either "ref" or "follows" +// set. If not otherwise specified, a "ref" will be generated to a +// 'type="indirect"' flake, which is treated as simply the name of a +// flake to be resolved in the registry. + struct FlakeInput { std::optional ref; - bool isFlake = true; + bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path std::optional follows; bool absolute = false; // whether 'follows' is relative to the flake root FlakeInputs overrides; }; +// The Flake structure is the main internal representation of a flake.nix file. + struct Flake { - FlakeRef originalRef; - FlakeRef resolvedRef; - FlakeRef lockedRef; + FlakeRef originalRef; // the original flake specification (by the user) + FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake + FlakeRef lockedRef; // the specific local store result of invoking the fetcher std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index f4eb825a6..ac68cde0e 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,10 +12,25 @@ class Store; typedef std::string FlakeId; +// The FlakeRef represents a local nix store reference to a flake +// input for a Flake (it may be helpful to think of this object by the +// alternate name of "InputRefForFlake"). It is constructed by +// starting with an input description (usually the attrs or a url from +// the flake file), locating a fetcher for that input, and then +// capturing the Input object that fetcher generates (usually via +// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). +// +// The actual fetch not have been performed yet (i.e. a FlakeRef may +// be lazy), but the fetcher can be invoked at any time via the +// FlakeRef to ensure the store is populated with this input. + struct FlakeRef { + // fetcher-specific representation of the input, sufficient to + // perform the fetch operation. fetchers::Input input; + // sub-path within the fetched input that represents this input Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 89b1e6e7d..f361edf17 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,6 +21,13 @@ struct Tree struct InputScheme; +// The Input object is generated by a specific fetcher, based on the +// user-supplied input attribute in the flake.nix file, and contians +// the information that the specific fetcher needs to perform the +// actual fetch. The Input object is most commonly created via the +// "fromURL()" or "fromAttrs()" static functions which are provided the +// url or attrset specified in the flake file. + struct Input { friend struct InputScheme; @@ -82,6 +89,15 @@ public: std::optional getLastModified() const; }; + +// The InputScheme represents a type of fetcher. Each fetcher +// registers with nix at startup time. When processing an input for a +// flake, each scheme is given an opportunity to "recognize" that +// input from the url or attributes in the flake file's specification +// and return an Input object to represent the input if it is +// recognized. The Input object contains the information the fetcher +// needs to actually perform the "fetch()" when called. + struct InputScheme { virtual std::optional inputFromURL(const ParsedURL & url) = 0; From bcb3da3b6b510bd24f2bc973b39bf43d92fad7ce Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 08:58:14 -0700 Subject: [PATCH 017/477] Fix spelling error. --- src/libfetchers/fetchers.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index f361edf17..6aa8a3422 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -22,7 +22,7 @@ struct Tree struct InputScheme; // The Input object is generated by a specific fetcher, based on the -// user-supplied input attribute in the flake.nix file, and contians +// user-supplied input attribute in the flake.nix file, and contains // the information that the specific fetcher needs to perform the // actual fetch. The Input object is most commonly created via the // "fromURL()" or "fromAttrs()" static functions which are provided the From 5ae164b7cf2dd7ca1846f349b57131913aa7cf55 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:23:05 -0700 Subject: [PATCH 018/477] Update description of FlakeRef, incorporating suggestion. --- src/libexpr/flake/flakeref.hh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index ac68cde0e..2596dd18e 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,12 +12,19 @@ class Store; typedef std::string FlakeId; -// The FlakeRef represents a local nix store reference to a flake -// input for a Flake (it may be helpful to think of this object by the -// alternate name of "InputRefForFlake"). It is constructed by -// starting with an input description (usually the attrs or a url from -// the flake file), locating a fetcher for that input, and then -// capturing the Input object that fetcher generates (usually via +// A flake reference specifies how to fetch a flake or raw source +// (e.g. from a Git repository). It is created from a URL-like syntax +// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ +// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local +// path. +// +// Each flake will have a number of FlakeRef objects: one for each +// input to the flake. +// +// The normal method of constructing a FlakeRef is by starting with an +// input description (usually the attrs or a url from the flake file), +// locating a fetcher for that input, and then capturing the Input +// object that fetcher generates (usually via // FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). // // The actual fetch not have been performed yet (i.e. a FlakeRef may From 128c98ab0961ba234774508663f591758d3a2178 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:34:23 -0700 Subject: [PATCH 019/477] Clarification in the description of the FlakeInput. --- src/libexpr/flake/flake.hh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 40476a137..44cf636b5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,15 +17,16 @@ struct FlakeInput; typedef std::map FlakeInputs; -// FlakeInput is the flake-level parsed form of the "input" entries in +// FlakeInput is the 'Flake'-level parsed form of the "input" entries in // the flake file. // -// A FlakeInput is normally constructed by initially -// first constructing a FlakeRef (a fetcher, the fetcher-specific -// representation of the input specification, and the fetched local -// store path result) and then creating this FlakeInput to hold that -// FlakeRef, along with anything that might override that FlakeRef -// (like command-line overrides or "follows" specifications). +// A FlakeInput is normally constructed by the 'parseFlakeInput' +// function which parses the input specification in the '.flake' file +// to create a 'FlakeRef' (a fetcher, the fetcher-specific +// representation of the input specification, and possibly the fetched +// local store path result) and then creating this FlakeInput to hold +// that FlakeRef, along with anything that might override that +// FlakeRef (like command-line overrides or "follows" specifications). // // A FlakeInput is also sometimes constructed directly from a FlakeRef // instead of starting at the flake-file input specification From 887be7b6f233822f03d156dafa156276d59162fd Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Mon, 28 Sep 2020 09:37:26 -0700 Subject: [PATCH 020/477] Switch comment format from '// ...' to '/* ... */' for consistency. --- src/libexpr/flake/flake.hh | 39 +++++++++++++++---------------- src/libexpr/flake/flakeref.hh | 43 ++++++++++++++++++----------------- src/libfetchers/fetchers.hh | 28 ++++++++++++----------- 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 44cf636b5..cf62c7741 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -17,25 +17,26 @@ struct FlakeInput; typedef std::map FlakeInputs; -// FlakeInput is the 'Flake'-level parsed form of the "input" entries in -// the flake file. -// -// A FlakeInput is normally constructed by the 'parseFlakeInput' -// function which parses the input specification in the '.flake' file -// to create a 'FlakeRef' (a fetcher, the fetcher-specific -// representation of the input specification, and possibly the fetched -// local store path result) and then creating this FlakeInput to hold -// that FlakeRef, along with anything that might override that -// FlakeRef (like command-line overrides or "follows" specifications). -// -// A FlakeInput is also sometimes constructed directly from a FlakeRef -// instead of starting at the flake-file input specification -// (e.g. overrides, follows, and implicit inputs). -// -// A FlakeInput will usually have one of either "ref" or "follows" -// set. If not otherwise specified, a "ref" will be generated to a -// 'type="indirect"' flake, which is treated as simply the name of a -// flake to be resolved in the registry. +/* FlakeInput is the 'Flake'-level parsed form of the "input" entries + * in the flake file. + * + * A FlakeInput is normally constructed by the 'parseFlakeInput' + * function which parses the input specification in the '.flake' file + * to create a 'FlakeRef' (a fetcher, the fetcher-specific + * representation of the input specification, and possibly the fetched + * local store path result) and then creating this FlakeInput to hold + * that FlakeRef, along with anything that might override that + * FlakeRef (like command-line overrides or "follows" specifications). + * + * A FlakeInput is also sometimes constructed directly from a FlakeRef + * instead of starting at the flake-file input specification + * (e.g. overrides, follows, and implicit inputs). + * + * A FlakeInput will usually have one of either "ref" or "follows" + * set. If not otherwise specified, a "ref" will be generated to a + * 'type="indirect"' flake, which is treated as simply the name of a + * flake to be resolved in the registry. + */ struct FlakeInput { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 2596dd18e..0292eb210 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -12,32 +12,33 @@ class Store; typedef std::string FlakeId; -// A flake reference specifies how to fetch a flake or raw source -// (e.g. from a Git repository). It is created from a URL-like syntax -// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ -// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local -// path. -// -// Each flake will have a number of FlakeRef objects: one for each -// input to the flake. -// -// The normal method of constructing a FlakeRef is by starting with an -// input description (usually the attrs or a url from the flake file), -// locating a fetcher for that input, and then capturing the Input -// object that fetcher generates (usually via -// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). -// -// The actual fetch not have been performed yet (i.e. a FlakeRef may -// be lazy), but the fetcher can be invoked at any time via the -// FlakeRef to ensure the store is populated with this input. +/* A flake reference specifies how to fetch a flake or raw source + * (e.g. from a Git repository). It is created from a URL-like syntax + * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ + * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local + * path. + * + * Each flake will have a number of FlakeRef objects: one for each + * input to the flake. + * + * The normal method of constructing a FlakeRef is by starting with an + * input description (usually the attrs or a url from the flake file), + * locating a fetcher for that input, and then capturing the Input + * object that fetcher generates (usually via + * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). + * + * The actual fetch not have been performed yet (i.e. a FlakeRef may + * be lazy), but the fetcher can be invoked at any time via the + * FlakeRef to ensure the store is populated with this input. + */ struct FlakeRef { - // fetcher-specific representation of the input, sufficient to - // perform the fetch operation. + /* fetcher-specific representation of the input, sufficient to + perform the fetch operation. */ fetchers::Input input; - // sub-path within the fetched input that represents this input + /* sub-path within the fetched input that represents this input */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6aa8a3422..84c954899 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,12 +21,13 @@ struct Tree struct InputScheme; -// The Input object is generated by a specific fetcher, based on the -// user-supplied input attribute in the flake.nix file, and contains -// the information that the specific fetcher needs to perform the -// actual fetch. The Input object is most commonly created via the -// "fromURL()" or "fromAttrs()" static functions which are provided the -// url or attrset specified in the flake file. +/* The Input object is generated by a specific fetcher, based on the + * user-supplied input attribute in the flake.nix file, and contains + * the information that the specific fetcher needs to perform the + * actual fetch. The Input object is most commonly created via the + * "fromURL()" or "fromAttrs()" static functions which are provided + * the url or attrset specified in the flake file. + */ struct Input { @@ -90,13 +91,14 @@ public: }; -// The InputScheme represents a type of fetcher. Each fetcher -// registers with nix at startup time. When processing an input for a -// flake, each scheme is given an opportunity to "recognize" that -// input from the url or attributes in the flake file's specification -// and return an Input object to represent the input if it is -// recognized. The Input object contains the information the fetcher -// needs to actually perform the "fetch()" when called. +/* The InputScheme represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an input for a + * flake, each scheme is given an opportunity to "recognize" that + * input from the url or attributes in the flake file's specification + * and return an Input object to represent the input if it is + * recognized. The Input object contains the information the fetcher + * needs to actually perform the "fetch()" when called. + */ struct InputScheme { From 59f2dd8e8da1f82aa9e29e30ba1df643434a9254 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 6 Oct 2020 20:08:51 +0200 Subject: [PATCH 021/477] libfetchers/github: allow slashes in refs Refs #4061 --- src/libfetchers/github.cc | 20 +++++++++++++++++--- src/libutil/url-parts.hh | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..3d1cc15e2 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme std::optional ref; std::optional host_url; - if (path.size() == 2) { - } else if (path.size() == 3) { + auto size = path.size(); + if (size == 3) { if (std::regex_match(path[2], revRegex)) rev = Hash::parseAny(path[2], htSHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else + } else if (size > 3) { + std::string rs; + for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { + rs += *i; + if (std::next(i) != path.end()) { + rs += "/"; + } + } + + if (std::regex_match(rs, refRegex)) { + ref = rs; + } else { + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + } + } else if (size < 2) throw BadURL("URL '%s' is invalid", url.url); for (auto &[name, value] : url.query) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 68be15cb0..e0e2809fd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -22,7 +22,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; // A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check extern std::regex refRegex; // Instead of defining what a good Git Ref is, we define what a bad Git Ref is From 5c74a6147b4b81dc5b173f190f02f6681ec4b0fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 17:07:14 +0000 Subject: [PATCH 022/477] Properly type the derivation and substitution goal maps As a bonus, Worker::removeGoal is less inefficient. --- src/libstore/build.hh | 11 +++++------ src/libstore/build/worker.cc | 33 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/build.hh b/src/libstore/build.hh index 8027c61b1..894feab5b 100644 --- a/src/libstore/build.hh +++ b/src/libstore/build.hh @@ -28,7 +28,6 @@ struct HookInstance; /* A pointer to a goal. */ struct Goal; -class DerivationGoal; typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -140,6 +139,8 @@ struct Child steady_time_point timeStarted; }; +class DerivationGoal; +class SubstitutionGoal; /* The worker class. */ class Worker @@ -167,8 +168,8 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; + std::map> derivationGoals; + std::map> substitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -242,7 +243,7 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -305,8 +306,6 @@ public: typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -class SubstitutionGoal; - /* Unless we are repairing, we don't both to test validity and just assume it, so the choices are `Absent` or `Valid`. */ enum struct PathStatus { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2fc9f6982..47403580e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -39,16 +39,13 @@ std::shared_ptr Worker::makeDerivationGoalCommon( const StringSet & wantedOutputs, std::function()> mkDrvGoal) { - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { + std::weak_ptr & goal_weak = derivationGoals[drvPath]; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { goal = mkDrvGoal(); - abstract_goal_weak = goal; + goal_weak = goal; wakeUp(goal); } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); goal->addWantedOutputs(wantedOutputs); } return goal; @@ -73,10 +70,10 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME + std::weak_ptr & goal_weak = substitutionGoals[path]; + auto goal = goal_weak.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; @@ -85,14 +82,14 @@ GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, return goal; } - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); + for (typename std::map>::iterator i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; + typename std::map>::iterator j = i; ++j; goalMap.erase(i); i = j; } @@ -102,8 +99,12 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) void Worker::removeGoal(GoalPtr goal) { - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, derivationGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, substitutionGoals); + else + assert(false); if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals From 55592b253f3dddb121c1072ca584e95c37729b6d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 18:04:24 +0000 Subject: [PATCH 023/477] Add some more docs --- src/libstore/build/worker.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 07c0c0f16..f8bacc514 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -11,6 +11,11 @@ namespace nix { class DerivationGoal; class SubstitutionGoal; +/* Workaround for not being able to declare a something like + + class SubstitutionGoal : public Goal; + + even when Goal is a complete type; */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; From f6ed1a96b397f0345af029127cfde86bcd0247d2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 18:54:36 +0000 Subject: [PATCH 024/477] `build-static` -> `buildStatic` in Nix's flake --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index a5710f608..0602861fa 100644 --- a/flake.nix +++ b/flake.nix @@ -228,7 +228,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); + buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -441,7 +441,7 @@ binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { - build-static = self.hydraJobs.build-static.${system}; + buildStatic = self.hydraJobs.buildStatic.${system}; }); packages = forAllSystems (system: { From 64be1c15c229facfa849f5667f603cce951a8488 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 19:05:06 +0000 Subject: [PATCH 025/477] Add missing include for MAX_PATH And remove one that we didn't actually need to add --- src/libstore/globals.cc | 1 - src/libutil/tests/tests.cc | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f4a4f348f..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 8e77ccbe1..ffba832d8 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "types.hh" +#include #include namespace nix { From 48ce62737750215208804947a1509ad7d26f6214 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 20:13:01 +0000 Subject: [PATCH 026/477] Make a better -lz hack Per the comments, the underlying issue is https://github.com/libarchive/libarchive/issues/1446, knowing this allows the hack to be much more targetted. --- configure.ac | 4 ++++ mk/programs.mk | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index eecb107d7..39306b953 100644 --- a/configure.ac +++ b/configure.ac @@ -179,6 +179,10 @@ AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) +# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed +if test "$shared" != yes; then + LIBARCHIVE_LIBS+=' -lz' +fi # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) diff --git a/mk/programs.mk b/mk/programs.mk index a96ff56af..3fa9685c3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) From 5cfdf16dd62dcfa2eb32b4e9ae390cb99d683907 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 17 Oct 2020 22:08:18 +0200 Subject: [PATCH 027/477] Convert VM tests to Python Perl-based tests are deprecated since NixOS 20.03 and subsequently got removed in NixOS 20.09, which effectively means that tests are going to fail as soon as we build it with NixOS 20.09 or anything newer. I've put "# fmt: off" at the start of every testScript, because formatting with Black really messes up indentation and I don't think it really adds anything in value or readability for inlined Python scripts. Signed-off-by: aszlig --- tests/github-flakes.nix | 45 +++++------ tests/nix-copy-closure.nix | 69 +++++++++-------- tests/remote-builds.nix | 70 ++++++++--------- tests/setuid.nix | 149 ++++++++++++++++++++----------------- 4 files changed, 178 insertions(+), 155 deletions(-) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index a47610d9a..2de3e2bc0 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -1,6 +1,6 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -113,36 +113,37 @@ makeTest ( }; }; - testScript = { nodes }: - '' - use POSIX qw(strftime); + testScript = { nodes }: '' + # fmt: off + import json + import time - startAll; + start_all() - $github->waitForUnit("httpd.service"); + github.wait_for_unit("httpd.service") - $client->succeed("curl -v https://github.com/ >&2"); + client.succeed("curl -v https://github.com/ >&2") + client.succeed("nix registry list | grep nixpkgs") - $client->succeed("nix registry list | grep nixpkgs"); + rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" - $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n" - or die "revision mismatch"; + client.succeed("nix registry pin nixpkgs") - $client->succeed("nix registry pin nixpkgs"); + client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") - $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2"); + # Shut down the web server. The flake should be cached on the client. + github.succeed("systemctl stop httpd.service") - # Shut down the web server. The flake should be cached on the client. - $github->succeed("systemctl stop httpd.service"); + info = json.loads(client.succeed("nix flake info nixpkgs --json")) + date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) + assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" - my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified"); - strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModifiedDate}" or die "time mismatch"; + client.succeed("nix build nixpkgs#hello") - $client->succeed("nix build nixpkgs#hello"); - - # The build shouldn't fail even with --tarball-ttl 0 (the server - # being down should not be a fatal error). - $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0"); - ''; + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") + ''; }) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 9c9d119b7..68f9c70b3 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -25,41 +25,46 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the server. - $server->succeed("mkdir -m 700 /root/.ssh"); - $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $server->waitForUnit("sshd"); - $client->waitForUnit("network.target"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'"); + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) - # Copy the closure of package A from the client to the server. - $server->fail("nix-store --check-validity ${pkgA}"); - $client->succeed("nix-copy-closure --to server --gzip ${pkgA} >&2"); - $server->succeed("nix-store --check-validity ${pkgA}"); + client.succeed("mkdir -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Copy the closure of package B from the server to the client. - $client->fail("nix-store --check-validity ${pkgB}"); - $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2"); - $client->succeed("nix-store --check-validity ${pkgB}"); + # Install the SSH key on the server. + server.succeed("mkdir -m 700 /root/.ssh") + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") - # Copy the closure of package C via the SSH substituter. - $client->fail("nix-store -r ${pkgC}"); - # FIXME - #$client->succeed( - # "nix-store --option use-ssh-substituter true" - # . " --option ssh-substituter-hosts root\@server" - # . " -r ${pkgC} >&2"); - #$client->succeed("nix-store --check-validity ${pkgC}"); - ''; + # Copy the closure of package A from the client to the server. + server.fail("nix-store --check-validity ${pkgA}") + client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2") + server.succeed("nix-store --check-validity ${pkgA}") + # Copy the closure of package B from the server to the client. + client.fail("nix-store --check-validity ${pkgB}") + client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the closure of package C via the SSH substituter. + client.fail("nix-store -r ${pkgC}") + # FIXME + # client.succeed( + # "nix-store --option use-ssh-substituter true" + # " --option ssh-substituter-hosts root\@server" + # " -r ${pkgC} >&2" + # ) + # client.succeed("nix-store --check-validity ${pkgC}") + ''; }) diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 153956619..305c82394 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -66,44 +66,46 @@ in }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -p -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the builders. - $client->waitForUnit("network.target"); - foreach my $builder ($builder1, $builder2) { - $builder->succeed("mkdir -p -m 700 /root/.ssh"); - $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $builder->waitForUnit("sshd"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); - } + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Perform a build and check that it was performed on the builder. - my $out = $client->succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ); - $builder1->succeed("test -e $out"); + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - # And a parallel build. - my ($out1, $out2) = split /\s/, - $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); - $builder1->succeed("test -e $out1 -o -e $out2"); - $builder2->succeed("test -e $out1 -o -e $out2"); + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") - # And a failing build. - $client->fail("nix-build ${expr nodes.client.config 5}"); + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") - # Test whether the build hook automatically skips unavailable builders. - $builder1->block; - $client->succeed("nix-build ${expr nodes.client.config 4}"); - ''; + # And a failing build. + client.fail("nix-build ${expr nodes.client.config 5}") + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client.config 4}") + ''; }) diff --git a/tests/setuid.nix b/tests/setuid.nix index 6f2f7d392..8d31a18be 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -2,7 +2,7 @@ { nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; @@ -17,94 +17,109 @@ makeTest { virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + start_all() - # Copying to /tmp should succeed. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - ")\' '); + # Copying to /tmp should succeed. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setuid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 4755 /tmp/id - ")\' '); + # Creating a setuid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setgid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # Creating a setgid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The checks should also work on 32-bit binaries. - $machine->fail('nix-build --no-sandbox -E \'(with import { system = "i686-linux"; }; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # The checks should also work on 32-bit binaries. + machine.fail(r""" + nix-build --no-sandbox -E '(with import { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The tests above use fchmodat(). Test chmod() as well. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 0666, qw(/tmp/id) or die\" - ")\' '); + # The tests above use fchmodat(). Test chmod() as well. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 04755, qw(/tmp/id) or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # And test fchmod(). - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" - ")\' '); + # And test fchmod(). + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); - - $machine->succeed("rm /tmp/id"); - ''; + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + machine.succeed("rm /tmp/id") + ''; } From cfa26cf18135ec1a16ce72319b0bc8b297600ef1 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 17 Oct 2020 23:34:38 +0200 Subject: [PATCH 028/477] tests: Add names to VM tests Having vm-test-run-unnamed for all the test derivation doesn't look very nice, so in order to better distinguish them from their store path, let's actually give them proper names. Signed-off-by: aszlig --- tests/github-flakes.nix | 1 + tests/nix-copy-closure.nix | 1 + tests/remote-builds.nix | 1 + tests/setuid.nix | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index 2de3e2bc0..7ac397d81 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -64,6 +64,7 @@ in makeTest ( { + name = "github-flakes"; nodes = { # Impersonate github.com and api.github.com. diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 68f9c70b3..e5f6a0f12 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { }; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { + name = "nix-copy-closure"; nodes = { client = diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 305c82394..b9e7352c0 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -36,6 +36,7 @@ let in { + name = "remote-builds"; nodes = { builder1 = builder; diff --git a/tests/setuid.nix b/tests/setuid.nix index 8d31a18be..35eb304ed 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { }; makeTest { + name = "setuid"; machine = { config, lib, pkgs, ... }: From e6f8ae56d82813edbcf09fda58305de47369f964 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:45:31 +0000 Subject: [PATCH 029/477] tab -> space --- src/libstore/build/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 97ffb9a34..4d3df26f3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -455,7 +455,7 @@ void Worker::markContentsGood(const StorePath & path) GoalPtr upcast_goal(std::shared_ptr subGoal) { - return subGoal; + return subGoal; } } From 57d0432b395cc4d70792d2df5794ff2e0dd02d3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:47:52 +0000 Subject: [PATCH 030/477] Just use `auto` in two places. --- src/libstore/build/worker.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4d3df26f3..17c10cd71 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -90,10 +90,10 @@ template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (typename std::map>::iterator i = goalMap.begin(); + for (auto i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - typename std::map>::iterator j = i; ++j; + auto j = i; ++j; goalMap.erase(i); i = j; } From 7ed46c15744461534478e2ed0aa25a2b2e536c6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:50:12 +0000 Subject: [PATCH 031/477] Explain that `upcast_goal` is still a static cast --- src/libstore/build/worker.hh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index f8bacc514..3a53a8def 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -15,7 +15,11 @@ class SubstitutionGoal; class SubstitutionGoal : public Goal; - even when Goal is a complete type; */ + even when Goal is a complete type. + + This is still a static cast. The purpose of exporting it is to define it in + a place where `SubstitutionGoal` is concrete, and use it in a place where it + is opaque. */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; From a53438d18f1faddb9f15c6980c828eb11820284c Mon Sep 17 00:00:00 2001 From: Stefan Frijters Date: Sun, 18 Oct 2020 14:14:37 +0200 Subject: [PATCH 032/477] doc: nix-shell in pure mode does *not* source user bashrc --- doc/manual/src/command-ref/nix-shell.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 45a5ff08c..d1266930e 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -76,8 +76,8 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. Note that `~/.bashrc` and (depending on your Bash - installation) `/etc/bashrc` are still sourced, so any variables set + retained. Note that (depending on your Bash + installation) `/etc/bashrc` is still sourced, so any variables set there will affect the interactive shell. - `--packages` / `-p` *packages*… From 94f1e4a441ee64bcc7a961d941ec901de750d880 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:26:23 +0200 Subject: [PATCH 033/477] Typo --- src/libstore/build/goal.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 360c160ce..0781a9d38 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this unsigned int nrNoSubstituters; /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ + failed because they had unsubstitutable references. */ unsigned int nrIncompleteClosure; /* Name of this goal for debugging purposes. */ From ea8d32020e1e24576cf5e8ff1a835f149dbd81c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 00:43:52 +0200 Subject: [PATCH 034/477] Tests for #3964 --- tests/binary-cache.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index fe4ddec8d..e14cf882e 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -239,3 +239,37 @@ nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath diff -u \ <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) + +# Test against issue https://github.com/NixOS/nix/issues/3964 +# +expr=' + with import ./config.nix; + mkDerivation { + name = "multi-output"; + buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref"; + outputs = ["out" "doc"]; + } +' +outPath=$(nix-build --no-out-link -E "$expr") +docPath=$(nix-store -q --references $outPath) + +# $ nix-store -q --tree $outPath +# ...-multi-output +# +---...-multi-output-doc + +nix copy --to "file://$cacheDir" $outPath +( echo $outPath $docPath + find $cacheDir +) >/tmp/blurb + +hashpart() { + basename "$1" | cut -c1-32 +} + +# break the closure of out by removing doc +rm $cacheDir/$(hashpart $docPath).narinfo + +nix-store --delete $outPath $docPath +# -vvv is the level that logs during the loop +timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ + --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From bd9eb5c743faf1b3c33f4e1c2ccf317977d4be9d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:21:53 +0200 Subject: [PATCH 035/477] DerivationGoal: only retry if output closure incomplete is only problem --- src/libstore/build/derivation-goal.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fda05f0e9..1c9217537 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -330,8 +330,13 @@ void DerivationGoal::outputsSubstitutionTried() /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; From 93bd014c8c416580d8eeaa487e180d6c2549028c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 20:32:46 +0200 Subject: [PATCH 036/477] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index b087cd8d5..c51582cf0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-copy-closure /scripts/nix-reduce-build /scripts/nix-http-export.cgi /scripts/nix-profile-daemon.sh From 20a7d8d23a19530207378fc74ddd558232240f43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 20:32:59 +0200 Subject: [PATCH 037/477] Add some missing clean-files --- misc/systemd/local.mk | 2 ++ misc/upstart/local.mk | 2 ++ 2 files changed, 4 insertions(+) diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 004549fd2..785db52a4 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -2,4 +2,6 @@ ifeq ($(OS), Linux) $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) + clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service + endif diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index a73dc061e..5071676dc 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -2,4 +2,6 @@ ifeq ($(OS), Linux) $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) + clean-files += $(d)/nix-daemon.conf + endif From 532d2bc1890d59a31e212eaea8d351a8cca66e19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:20:35 +0200 Subject: [PATCH 038/477] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3a10a004bb5802d5f23c58886722e4239705e733' -> 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31' --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index ddff58979..9f8c788ac 100644 --- a/flake.lock +++ b/flake.lock @@ -18,16 +18,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1602604700, - "narHash": "sha256-TSfAZX0czPf1P8xnnGFXcoeoM9I5CaFjAdNP63W9DCY=", + "lastModified": 1602702596, + "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a10a004bb5802d5f23c58886722e4239705e733", + "rev": "ad0d20345219790533ebe06571f82ed6b034db31", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-20.03-small", + "ref": "nixos-20.09-small", "type": "indirect" } }, From 62cf1d815a6c55853e9a4041e85b7e6e0b04d7e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:31:27 +0200 Subject: [PATCH 039/477] Switch to Nixpkgs 20.09 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 0602861fa..22ea2911e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small"; + inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: From c27fcd94ce3de918d506afe90759d97d1946eca1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:44:07 +0200 Subject: [PATCH 040/477] Remove buildStatic from checks checks should be relatively fast, but buildStatic depends on a lot of stuff that isn't in the binary cache (e.g. musl builds of Git and Mercurial that we probably don't need since we don't link against them...). --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 22ea2911e..abc614c8d 100644 --- a/flake.nix +++ b/flake.nix @@ -440,8 +440,6 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; - } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { - buildStatic = self.hydraJobs.buildStatic.${system}; }); packages = forAllSystems (system: { From 9c3dc9d7ca11555fbafe77e9ca8ed9fc214ab2f8 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 7 Sep 2020 23:53:31 -0500 Subject: [PATCH 041/477] update macOS version handling for Big Sur Keeping this commit narrow for reviewability, but some of these conditionals will change in subsequent commits in this PR. Fixes #3852. --- scripts/install-nix-from-closure.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 6efd8af18..e7e063007 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -24,9 +24,11 @@ fi # macOS support for 10.12.6 or higher if [ "$(uname -s)" = "Darwin" ]; then - macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) - macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) - if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + IFS='.' read macos_major macos_minor macos_patch << EOF +$(sw_vers -productVersion) +EOF + if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + # patch may not be present; command substitution for simplicity echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi @@ -88,7 +90,7 @@ while [ $# -gt 0 ]; do ) >&2 # darwin and Catalina+ - if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then + if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then ( echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" echo " store and mount it at /nix. This is the recommended way to create" @@ -109,7 +111,7 @@ if [ "$(uname -s)" = "Darwin" ]; then fi info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) - if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then + if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." From 1f02b65c590b5a33c1b70dba186eacfe3f67f149 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 00:07:53 -0500 Subject: [PATCH 042/477] fix xpath and conditional bugs; xpath -> xmllint - xpath -> xmllint: xpath's cli interface changed in Big Sur rather than add conditional logic for picking the correct syntax for xpath, I'm changing to xmllint --xpath, which appears to be consistent across versions I've tested... - /plist/dict/key[text()='Writable']/following-sibling::true[1] doesn't do quite what's expected. It was written to try to select a node paired with the Writable key, but it will also select the *next* node that appears even if it was paired with another key. - I think there's also a logic bug in the conditionals here. I'm not sure anyone ever actuall saw it, thanks to the xpath bug, though. With the xpath fix, this conditional passes if /nix does not exist, / IS writable, and the version is Catalina+. I think it meant to test for /nix does not exist, / is NOT writable, and the version is Catalina+. I reworked this lightly to make it a little clearer at the code level. --- scripts/install-nix-from-closure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index e7e063007..4e64f3d43 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -110,8 +110,8 @@ if [ "$(uname -s)" = "Darwin" ]; then "$self/create-darwin-volume.sh" fi - info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) - if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then + writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)" + if ! [ -e $dest ] && [ "$writable" = "false" ]; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." From e736f8f6e44180d7ed7cc1975b48c603c6c4f611 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 00:45:27 -0500 Subject: [PATCH 043/477] replace xpath with xmllint --xpath; simplify As mentioned in previous commit, Big Sur changes the syntax for the xpath command slightly. In the process of testing out replacements for these, I noticed a few small simplification wins. --- scripts/create-darwin-volume.sh | 46 ++++++--------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index dac30d72d..8c4558c7f 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -5,42 +5,13 @@ root_disk() { diskutil info -plist / } -apfs_volumes_for() { - disk=$1 - diskutil apfs list -plist "$disk" -} - -disk_identifier() { - xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null -} - -volume_list_true() { - key=$1 - xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null -} - -volume_get_string() { - key=$1 i=$2 - xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null +# i.e., "disk1" +root_disk_identifier() { + diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - } find_nix_volume() { - disk=$1 - i=1 - volumes=$(apfs_volumes_for "$disk") - while true; do - name=$(echo "$volumes" | volume_get_string "Name" "$i") - if [ -z "$name" ]; then - break - fi - case "$name" in - [Nn]ix*) - echo "$name" - break - ;; - esac - i=$((i+1)) - done + diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true } test_fstab() { @@ -89,9 +60,7 @@ test_t2_chip_present(){ } test_filevault_in_use() { - disk=$1 - # list vols on disk | get value of Filevault key | value is true - apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true + fdesetup isactive >/dev/null } # use after error msg for conditions we don't understand @@ -143,12 +112,12 @@ main() { fi fi - disk=$(root_disk | disk_identifier) + disk="$(root_disk_identifier)" volume=$(find_nix_volume "$disk") if [ -z "$volume" ]; then echo "Creating a Nix Store volume..." >&2 - if test_filevault_in_use "$disk"; then + if test_filevault_in_use; then # TODO: Not sure if it's in-scope now, but `diskutil apfs list` # shows both filevault and encrypted at rest status, and it # may be the more semantic way to test for this? It'll show @@ -178,6 +147,7 @@ main() { if ! test_fstab; then echo "Configuring /etc/fstab..." >&2 label=$(echo "$volume" | sed 's/ /\\040/g') + # shellcheck disable=SC2209 printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs fi } From fe807904e5e6e56b551f34f3586e69ea6498287c Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 8 Sep 2020 01:01:11 -0500 Subject: [PATCH 044/477] adapt to apfs.util flag diff in catalina/big sur Fixes #3957. Just runs both forms to minimize moving parts. --- scripts/create-darwin-volume.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 8c4558c7f..32fa577a8 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -26,6 +26,20 @@ test_synthetic_conf() { grep -q "^nix$" /etc/synthetic.conf 2>/dev/null } +# Create the paths defined in synthetic.conf, saving us a reboot. +create_synthetic_objects(){ + # Big Sur takes away the -B flag we were using and replaces it + # with a -t flag that appears to do the same thing (but they + # don't behave exactly the same way in terms of return values). + # This feels a little dirty, but as far as I can tell the + # simplest way to get the right one is to just throw away stderr + # and call both... :] + { + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true # Big Sur + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true # Catalina + } >/dev/null 2>&1 +} + test_nix() { test -d "/nix" } @@ -101,7 +115,7 @@ main() { if ! test_nix; then echo "Creating mountpoint for /nix..." >&2 - /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true + create_synthetic_objects # the ones we defined in synthetic.conf if ! test_nix; then sudo mkdir -p /nix 2>/dev/null || true fi From 3a8699ac4ffc0d9b611c471e2668e6b22cc65767 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 10 Sep 2020 18:21:04 -0500 Subject: [PATCH 045/477] restore create-darwin-volume to release tarball The move from release.nix to flake.nix appears to have lost some changes from #3628 / 1c56f18a8122b605c28000e295d5e223f272cccd, leaving create-darwin-volume.sh out of the release tarball. Under the assumption that this was just an accident/byproduct of when flake.nix split off and not intentional, I am restoring those edits. --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index a50533a29..d0c3bfcd2 100644 --- a/flake.nix +++ b/flake.nix @@ -235,6 +235,7 @@ } '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} @@ -253,6 +254,7 @@ # SC1090: Don't worry about not being able to find # $nix/etc/profile.d/nix.sh shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh shellcheck $TMPDIR/install-darwin-multi-user.sh shellcheck $TMPDIR/install-systemd-multi-user.sh @@ -268,6 +270,7 @@ fi chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh chmod +x $TMPDIR/install-darwin-multi-user.sh chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user @@ -280,11 +283,15 @@ --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ $(cat ${installerClosureInfo}/store-paths) ''); From b719f686a8c8936fe831ce730f28638d8b1e2982 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 11 Sep 2020 12:06:01 -0500 Subject: [PATCH 046/477] fix skipped multi-user install steps on macOS Some of the changes in #3788 to support non-systemd Nix installs don't appear to be aware that the darwin installer exists, which resulted in some skipped steps and inappropriate instructions. --- scripts/install-darwin-multi-user.sh | 7 +++++ scripts/install-multi-user.sh | 34 +++----------------- scripts/install-systemd-multi-user.sh | 45 ++++++++++++++++++++------- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 49076bd5c..a27be2a43 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -37,6 +37,13 @@ poly_service_setup_note() { EOF } +poly_extra_try_me_commands(){ + : +} +poly_extra_setup_instructions(){ + : +} + poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon as a LaunchDaemon" \ cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index e5cc4d7ed..54edfe40d 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -71,11 +71,9 @@ uninstall_directions() { subheader "Uninstalling nix:" local step=0 - if [ -e /run/systemd/system ] && poly_service_installed_check; then + if poly_service_installed_check; then step=$((step + 1)) poly_service_uninstall_directions "$step" - else - step=$((step + 1)) fi for profile_target in "${PROFILE_TARGETS[@]}"; do @@ -255,40 +253,20 @@ function finish_success { echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." fi - if [ -e /run/systemd/system ]; then - cat < Date: Fri, 11 Sep 2020 16:45:58 -0500 Subject: [PATCH 047/477] create missing profile files to fix zsh envvars Env vars for ZSH were moved from /etc/zshrc to /etc/zshenv in #3608 to address an issue with zshrc getting clobbered by OS updates, but /etc/zshenv doesn't exist by default--so *nothing* would get set up for zsh users unless they already happened to have /etc/zshenv. Creating these files if they don't exist. Also cut separate creation of profile.d/nix.sh, which isn't needed now. --- scripts/install-multi-user.sh | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 54edfe40d..5e8b4ac18 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -608,24 +608,20 @@ EOF } configure_shell_profile() { - # If there is an /etc/profile.d directory, we want to ensure there - # is a nix.sh within it, so we can use the following loop to add - # the source lines to it. Note that I'm _not_ adding the source - # lines here, because we want to be using the regular machinery. - # - # If we go around that machinery, it becomes more complicated and - # adds complications to the uninstall instruction generator and - # old instruction sniffer as well. - if [ -d /etc/profile.d ]; then - _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \ - touch /etc/profile.d/nix.sh - fi - for profile_target in "${PROFILE_TARGETS[@]}"; do if [ -e "$profile_target" ]; then _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \ cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX" + else + # try to create the file if its directory exists + target_dir="$(dirname "$profile_target")" + if [ -d "$target_dir" ]; then + _sudo "to create a stub $profile_target which will be updated" \ + touch "$profile_target" + fi + fi + if [ -e "$profile_target" ]; then shell_source_lines \ | _sudo "extend your $profile_target with nix-daemon settings" \ tee -a "$profile_target" From f289bdb9d4e8432160c5dcdc037a930b2025d11b Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 19 Oct 2020 11:54:21 -0500 Subject: [PATCH 048/477] discourage casual Big Sur installs --- scripts/install-nix-from-closure.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 4e64f3d43..ea2e47b7f 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -27,6 +27,15 @@ if [ "$(uname -s)" = "Darwin" ]; then IFS='.' read macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF + # TODO: this is a temporary speed-bump to keep people from naively installing Nix + # on macOS Big Sur (11.0+, 10.16+) until nixpkgs updates are ready for them. + # *Ideally* this is gone before next Nix release. If you're intentionally working on + # Nix + Big Sur, just comment out this block and be on your way :) + if [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 15 ]; }; then + echo "$0: nixpkgs isn't quite ready to support macOS $(sw_vers -productVersion) yet" + exit 1 + fi + if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then # patch may not be present; command substitution for simplicity echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" From f6aaac2b5983bd87cd6e2f648c9dd835c6d9373c Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Tue, 20 Oct 2020 11:48:07 -0500 Subject: [PATCH 049/477] Make bash non-interactive for nix develop --phase Fix #3975: Currently if Ctrl-C is pressed during a phase, the interactive subshell is not exited. Removing --rcfile when --phase is present makes bash non-interactive --- src/nix/develop.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9372f43de..380417c82 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -368,7 +368,6 @@ struct CmdDevelop : Common, MixEnvironment // rid of that. script += fmt("foundMakefile=1\n"); script += fmt("runHook %1%Phase\n", *phase); - script += fmt("exit 0\n", *phase); } else if (!command.empty()) { @@ -408,7 +407,10 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } - auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + // If running a phase, don't want an interactive shell running after + // Ctrl-C, so don't pass --rcfile + auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath} + : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); restoreSignals(); From 39fbd3d82823ae6eadfbf91bb2b356caef90841c Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 21 Oct 2020 00:03:38 +0400 Subject: [PATCH 050/477] Fix iterating over $NIX_PROFILES in Zsh NIX_PROFILES is space separated list of directories, and passing it into for as is is considered to be 1-element list with the whole string. With shwordsplit option Zsh emulates other shells in this regard ans implicitely splits unquoted strings into words. Fixes #4167. --- scripts/nix-profile-daemon.sh.in | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 4bc7c6fc5..500a98992 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -17,11 +17,21 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES; do - if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then - export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + check_nix_profiles() { + if [ "$ZSH_VERSION" ]; then + # Zsh by default doesn't split words in unquoted parameter expansion. + # Set local_options for these options to be reverted at the end of the function + # and shwordsplit to force splitting words in $NIX_PROFILES below. + setopt local_options shwordsplit fi - done + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done + } + check_nix_profiles + unset -f check_nix_profiles fi export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" From 461cf2b85601e4510bff303e454059d80b9df8c0 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Mon, 19 Oct 2020 23:08:50 +0200 Subject: [PATCH 051/477] Add NIX_CONFIG env var for applying nix.conf overrides --- doc/manual/src/command-ref/conf-file-prefix.md | 3 +++ doc/manual/src/command-ref/env-common.md | 5 +++++ src/libstore/globals.cc | 6 ++++++ tests/config.sh | 9 +++++++++ 4 files changed, 23 insertions(+) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 9987393d2..d38456788 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -19,6 +19,9 @@ By default Nix reads settings from the following places: and `XDG_CONFIG_HOME`. If these are unset, it will look in `$HOME/.config/nix.conf`. + - If `NIX_OPTIONS` is set, its contents is treated as the contents of + a configuration file. + The configuration files consist of `name = value` pairs, one per line. Other files can be included with a line like `include diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 03016dba7..27e730fc8 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -81,6 +81,11 @@ Most Nix commands interpret the following environment variables: Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). + - `NIX_OPTIONS` + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + - `NIX_USER_CONF_FILES` Overrides the location of the user Nix configuration files to load from (defaults to the XDG spec locations). The variable is treated diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1238dc530..4df68d0c9 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -86,6 +86,12 @@ void loadConfFile() for (auto file = files.rbegin(); file != files.rend(); file++) { globalConfig.applyConfigFile(*file); } + + auto nixConfEnv = getEnv("NIX_CONFIG"); + if (nixConfEnv.has_value()) { + globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + } + } std::vector getUserConfigFiles() diff --git a/tests/config.sh b/tests/config.sh index 8fa349f11..eaa46c395 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -16,3 +16,12 @@ here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] + +# Test that it's possible to load config from the environment +prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" +exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +[[ $prev != $exp_cores ]] +[[ $exp_cores == "4242" ]] +[[ $exp_features == "nix-command flakes" ]] \ No newline at end of file From bdf2bcc989348fdcf223c9e2a383618454453eb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 20 Oct 2020 19:18:02 +0200 Subject: [PATCH 052/477] Remove conf-file.xml This was probably revived in a bad merge. --- doc/manual/command-ref/conf-file.xml | 1236 -------------------------- 1 file changed, 1236 deletions(-) delete mode 100644 doc/manual/command-ref/conf-file.xml diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml deleted file mode 100644 index d0f1b09ca..000000000 --- a/doc/manual/command-ref/conf-file.xml +++ /dev/null @@ -1,1236 +0,0 @@ - - - - - nix.conf - 5 - Nix - - - - - nix.conf - Nix configuration file - - -Description - -By default Nix reads settings from the following places: - -The system-wide configuration file -sysconfdir/nix/nix.conf -(i.e. /etc/nix/nix.conf on most systems), or -$NIX_CONF_DIR/nix.conf if -NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The -client assumes that the daemon has already loaded them. - - -User-specific configuration files: - - - If NIX_USER_CONF_FILES is set, then each path separated by - : will be loaded in reverse order. - - - - Otherwise it will look for nix/nix.conf files in - XDG_CONFIG_DIRS and XDG_CONFIG_HOME. - - The default location is $HOME/.config/nix.conf if - those environment variables are unset. - - -The configuration files consist of -name = -value pairs, one per line. Other -files can be included with a line like include -path, where -path is interpreted relative to the current -conf file and a missing file is an error unless -!include is used instead. -Comments start with a # character. Here is an -example configuration file: - - -keep-outputs = true # Nice for developers -keep-derivations = true # Idem - - -You can override settings on the command line using the - flag, e.g. --option keep-outputs -false. - -The following settings are currently available: - - - - - allowed-uris - - - - A list of URI prefixes to which access is allowed in - restricted evaluation mode. For example, when set to - https://github.com/NixOS, builtin functions - such as fetchGit are allowed to access - https://github.com/NixOS/patchelf.git. - - - - - - - allow-import-from-derivation - - By default, Nix allows you to import from a derivation, - allowing building at evaluation time. With this option set to false, Nix will throw an error - when evaluating an expression that uses this feature, allowing users to ensure their evaluation - will not require any builds to take place. - - - - - allow-new-privileges - - (Linux-specific.) By default, builders on Linux - cannot acquire new privileges by calling setuid/setgid programs or - programs that have file capabilities. For example, programs such - as sudo or ping will - fail. (Note that in sandbox builds, no such programs are available - unless you bind-mount them into the sandbox via the - option.) You can allow the - use of such programs by enabling this option. This is impure and - usually undesirable, but may be useful in certain scenarios - (e.g. to spin up containers or set up userspace network interfaces - in tests). - - - - - allowed-users - - - - A list of names of users (separated by whitespace) that - are allowed to connect to the Nix daemon. As with the - option, you can specify groups by - prefixing them with @. Also, you can allow - all users by specifying *. The default is - *. - - Note that trusted users are always allowed to connect. - - - - - - - auto-optimise-store - - If set to true, Nix - automatically detects files in the store that have identical - contents, and replaces them with hard links to a single copy. - This saves disk space. If set to false (the - default), you can still run nix-store - --optimise to get rid of duplicate - files. - - - - - builders - - A list of machines on which to perform builds. See for details. - - - - - builders-use-substitutes - - If set to true, Nix will instruct - remote build machines to use their own binary substitutes if available. In - practical terms, this means that remote hosts will fetch as many build - dependencies as possible from their own substitutes (e.g, from - cache.nixos.org), instead of waiting for this host to - upload them all. This can drastically reduce build times if the network - connection between this computer and the remote build host is slow. Defaults - to false. - - - - build-users-group - - This options specifies the Unix group containing - the Nix build user accounts. In multi-user Nix installations, - builds should not be performed by the Nix account since that would - allow users to arbitrarily modify the Nix store and database by - supplying specially crafted builders; and they cannot be performed - by the calling user since that would allow him/her to influence - the build result. - - Therefore, if this option is non-empty and specifies a valid - group, builds will be performed under the user accounts that are a - member of the group specified here (as listed in - /etc/group). Those user accounts should not - be used for any other purpose! - - Nix will never run two builds under the same user account at - the same time. This is to prevent an obvious security hole: a - malicious user writing a Nix expression that modifies the build - result of a legitimate Nix expression being built by another user. - Therefore it is good to have as many Nix build user accounts as - you can spare. (Remember: uids are cheap.) - - The build users should have permission to create files in - the Nix store, but not delete them. Therefore, - /nix/store should be owned by the Nix - account, its group should be the group specified here, and its - mode should be 1775. - - If the build users group is empty, builds will be performed - under the uid of the Nix process (that is, the uid of the caller - if NIX_REMOTE is empty, the uid under which the Nix - daemon runs if NIX_REMOTE is - daemon). Obviously, this should not be used in - multi-user settings with untrusted users. - - - - - - - compress-build-log - - If set to true (the default), - build logs written to /nix/var/log/nix/drvs - will be compressed on the fly using bzip2. Otherwise, they will - not be compressed. - - - - connect-timeout - - - - The timeout (in seconds) for establishing connections in - the binary cache substituter. It corresponds to - curl’s - option. - - - - - - - cores - - Sets the value of the - NIX_BUILD_CORES environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - enableParallelBuilding is set to - true, the builder passes the - flag to GNU Make. - It can be overridden using the command line switch and - defaults to 1. The value 0 - means that the builder should use all available CPU cores in the - system. - - See also . - - - diff-hook - - - Absolute path to an executable capable of diffing build results. - The hook executes if is - true, and the output of a build is known to not be the same. - This program is not executed to determine if two results are the - same. - - - - The diff hook is executed by the same user and group who ran the - build. However, the diff hook does not have write access to the - store path just built. - - - The diff hook program receives three parameters: - - - - - A path to the previous build's results - - - - - - A path to the current build's results - - - - - - The path to the build's derivation - - - - - - The path to the build's scratch directory. This directory - will exist only if the build was run with - . - - - - - - The stderr and stdout output from the diff hook will not be - displayed to the user. Instead, it will print to the nix-daemon's - log. - - - When using the Nix daemon, diff-hook must - be set in the nix.conf configuration file, and - cannot be passed at the command line. - - - - - - enforce-determinism - - See . - - - - extra-sandbox-paths - - A list of additional paths appended to - . Useful if you want to extend - its default value. - - - - - extra-platforms - - Platforms other than the native one which - this machine is capable of building for. This can be useful for - supporting additional architectures on compatible machines: - i686-linux can be built on x86_64-linux machines (and the default - for this setting reflects this); armv7 is backwards-compatible with - armv6 and armv5tel; some aarch64 machines can also natively run - 32-bit ARM code; and qemu-user may be used to support non-native - platforms (though this may be slow and buggy). Most values for this - are not enabled by default because build systems will often - misdetect the target platform and generate incompatible code, so you - may wish to cross-check the results of using this option against - proper natively-built versions of your - derivations. - - - - - extra-substituters - - Additional binary caches appended to those - specified in . When used by - unprivileged users, untrusted substituters (i.e. those not listed - in ) are silently - ignored. - - - - fallback - - If set to true, Nix will fall - back to building from source if a binary substitute fails. This - is equivalent to the flag. The - default is false. - - - - fsync-metadata - - If set to true, changes to the - Nix store metadata (in /nix/var/nix/db) are - synchronously flushed to disk. This improves robustness in case - of system crashes, but reduces performance. The default is - true. - - - - hashed-mirrors - - A list of web servers used by - builtins.fetchurl to obtain files by hash. - Given a hash type ht and a base-16 hash - h, Nix will try to download the file - from - hashed-mirror/ht/h. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the hashed mirror - http://tarballs.example.com/, when building the - derivation - - -builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} - - - Nix will attempt to download this file from - http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae - first. If it is not available there, if will try the original URI. - - - - - http-connections - - The maximum number of parallel TCP connections - used to fetch files from binary caches and by other downloads. It - defaults to 25. 0 means no limit. - - - - - keep-build-log - - If set to true (the default), - Nix will write the build log of a derivation (i.e. the standard - output and error of its builder) to the directory - /nix/var/log/nix/drvs. The build log can be - retrieved using the command nix-store -l - path. - - - - - keep-derivations - - If true (default), the garbage - collector will keep the derivations from which non-garbage store - paths were built. If false, they will be - deleted unless explicitly registered as a root (or reachable from - other roots). - - Keeping derivation around is useful for querying and - traceability (e.g., it allows you to ask with what dependencies or - options a store path was built), so by default this option is on. - Turn it off to save a bit of disk space (or a lot if - keep-outputs is also turned on). - - - keep-env-derivations - - If false (default), derivations - are not stored in Nix user environments. That is, the derivations of - any build-time-only dependencies may be garbage-collected. - - If true, when you add a Nix derivation to - a user environment, the path of the derivation is stored in the - user environment. Thus, the derivation will not be - garbage-collected until the user environment generation is deleted - (nix-env --delete-generations). To prevent - build-time-only dependencies from being collected, you should also - turn on keep-outputs. - - The difference between this option and - keep-derivations is that this one is - “sticky”: it applies to any user environment created while this - option was enabled, while keep-derivations - only applies at the moment the garbage collector is - run. - - - - keep-outputs - - If true, the garbage collector - will keep the outputs of non-garbage derivations. If - false (default), outputs will be deleted unless - they are GC roots themselves (or reachable from other roots). - - In general, outputs must be registered as roots separately. - However, even if the output of a derivation is registered as a - root, the collector will still delete store paths that are used - only at build time (e.g., the C compiler, or source tarballs - downloaded from the network). To prevent it from doing so, set - this option to true. - - - max-build-log-size - - - - This option defines the maximum number of bytes that a - builder can write to its stdout/stderr. If the builder exceeds - this limit, it’s killed. A value of 0 (the - default) means that there is no limit. - - - - - - max-free - - When a garbage collection is triggered by the - min-free option, it stops as soon as - max-free bytes are available. The default is - infinity (i.e. delete all garbage). - - - - max-jobs - - This option defines the maximum number of jobs - that Nix will try to build in parallel. The default is - 1. The special value auto - causes Nix to use the number of CPUs in your system. 0 - is useful when using remote builders to prevent any local builds (except for - preferLocalBuild derivation attribute which executes locally - regardless). It can be - overridden using the () - command line switch. - - See also . - - - - max-silent-time - - - - This option defines the maximum number of seconds that a - builder can go without producing any data on standard output or - standard error. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite - loop, or to catch remote builds that are hanging due to network - problems. It can be overridden using the command - line switch. - - The value 0 means that there is no - timeout. This is also the default. - - - - - - min-free - - - When free disk space in /nix/store - drops below min-free during a build, Nix - performs a garbage-collection until max-free - bytes are available or there is no more garbage. A value of - 0 (the default) disables this feature. - - - - - narinfo-cache-negative-ttl - - - - The TTL in seconds for negative lookups. If a store path is - queried from a substituter but was not found, there will be a - negative lookup cached in the local disk cache database for the - specified duration. - - - - - - narinfo-cache-positive-ttl - - - - The TTL in seconds for positive lookups. If a store path is - queried from a substituter, the result of the query will be cached - in the local disk cache database including some of the NAR - metadata. The default TTL is a month, setting a shorter TTL for - positive lookups can be useful for binary caches that have - frequent garbage collection, in which case having a more frequent - cache invalidation would prevent trying to pull the path again and - failing with a hash mismatch if the build isn't reproducible. - - - - - - - netrc-file - - If set to an absolute path to a netrc - file, Nix will use the HTTP authentication credentials in this file when - trying to download from a remote host through HTTP or HTTPS. Defaults to - $NIX_CONF_DIR/netrc. - - The netrc file consists of a list of - accounts in the following format: - - -machine my-machine -login my-username -password my-password - - - For the exact syntax, see the - curl documentation. - - This must be an absolute path, and ~ - is not resolved. For example, ~/.netrc won't - resolve to your home directory's .netrc. - - - - - - - plugin-files - - - A list of plugin files to be loaded by Nix. Each of these - files will be dlopened by Nix, allowing them to affect - execution through static initialization. In particular, these - plugins may construct static instances of RegisterPrimOp to - add new primops or constants to the expression language, - RegisterStoreImplementation to add new store implementations, - RegisterCommand to add new subcommands to the - nix command, and RegisterSetting to add new - nix config settings. See the constructors for those types for - more details. - - - Since these files are loaded into the same address space as - Nix itself, they must be DSOs compatible with the instance of - Nix running at the time (i.e. compiled against the same - headers, not linked to any incompatible libraries). They - should not be linked to any Nix libs directly, as those will - be available already at load time. - - - If an entry in the list is a directory, all files in the - directory are loaded as plugins (non-recursively). - - - - - - pre-build-hook - - - - - If set, the path to a program that can set extra - derivation-specific settings for this system. This is used for settings - that can't be captured by the derivation model itself and are too variable - between different versions of the same system to be hard-coded into nix. - - - The hook is passed the derivation path and, if sandboxes are enabled, - the sandbox directory. It can then modify the sandbox and send a series of - commands to modify various settings to stdout. The currently recognized - commands are: - - - - extra-sandbox-paths - - - - Pass a list of files and directories to be included in the - sandbox for this build. One entry per line, terminated by an empty - line. Entries have the same format as - sandbox-paths. - - - - - - - - - - - post-build-hook - - Optional. The path to a program to execute after each build. - - This option is only settable in the global - nix.conf, or on the command line by trusted - users. - - When using the nix-daemon, the daemon executes the hook as - root. If the nix-daemon is not involved, the - hook runs as the user executing the nix-build. - - - The hook executes after an evaluation-time build. - The hook does not execute on substituted paths. - The hook's output always goes to the user's terminal. - If the hook fails, the build succeeds but no further builds execute. - The hook executes synchronously, and blocks other builds from progressing while it runs. - - - The program executes with no arguments. The program's environment - contains the following environment variables: - - - - DRV_PATH - - The derivation for the built paths. - Example: - /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv - - - - - - OUT_PATHS - - Output paths of the built derivation, separated by a space character. - Example: - /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev - /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc - /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info - /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man - /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. - - - - - - See for an example - implementation. - - - - - repeat - - How many times to repeat builds to check whether - they are deterministic. The default value is 0. If the value is - non-zero, every build is repeated the specified number of - times. If the contents of any of the runs differs from the - previous ones and is - true, the build is rejected and the resulting store paths are not - registered as “valid” in Nix’s database. - - - require-sigs - - If set to true (the default), - any non-content-addressed path added or copied to the Nix store - (e.g. when substituting from a binary cache) must have a valid - signature, that is, be signed using one of the keys listed in - or - . Set to false - to disable signature checking. - - - - - restrict-eval - - - - If set to true, the Nix evaluator will - not allow access to any files outside of the Nix search path (as - set via the NIX_PATH environment variable or the - option), or to URIs outside of - . The default is - false. - - - - - - run-diff-hook - - - If true, enable the execution of . - - - - When using the Nix daemon, run-diff-hook must - be set in the nix.conf configuration file, - and cannot be passed at the command line. - - - - - sandbox - - If set to true, builds will be - performed in a sandboxed environment, i.e., - they’re isolated from the normal file system hierarchy and will - only see their dependencies in the Nix store, the temporary build - directory, private versions of /proc, - /dev, /dev/shm and - /dev/pts (on Linux), and the paths configured with the - sandbox-paths - option. This is useful to prevent undeclared dependencies - on files in directories such as /usr/bin. In - addition, on Linux, builds run in private PID, mount, network, IPC - and UTS namespaces to isolate them from other processes in the - system (except that fixed-output derivations do not run in private - network namespace to ensure they can access the network). - - Currently, sandboxing only work on Linux and macOS. The use - of a sandbox requires that Nix is run as root (so you should use - the “build users” - feature to perform the actual builds under different users - than root). - - If this option is set to relaxed, then - fixed-output derivations and derivations that have the - __noChroot attribute set to - true do not run in sandboxes. - - The default is true on Linux and - false on all other platforms. - - - - - - sandbox-dev-shm-size - - This option determines the maximum size of the - tmpfs filesystem mounted on - /dev/shm in Linux sandboxes. For the format, - see the description of the option of - tmpfs in - mount8. The - default is 50%. - - - - - - sandbox-paths - - A list of paths bind-mounted into Nix sandbox - environments. You can use the syntax - target=source - to mount a path in a different location in the sandbox; for - instance, /bin=/nix-bin will mount the path - /nix-bin as /bin inside the - sandbox. If source is followed by - ?, then it is not an error if - source does not exist; for example, - /dev/nvidiactl? specifies that - /dev/nvidiactl will only be mounted in the - sandbox if it exists in the host filesystem. - - Depending on how Nix was built, the default value for this option - may be empty or provide /bin/sh as a - bind-mount of bash. - - - - - secret-key-files - - A whitespace-separated list of files containing - secret (private) keys. These are used to sign locally-built - paths. They can be generated using nix-store - --generate-binary-cache-key. The corresponding public - key can be distributed to other users, who can add it to - in their - nix.conf. - - - - - show-trace - - Causes Nix to print out a stack trace in case of Nix - expression evaluation errors. - - - - - substitute - - If set to true (default), Nix - will use binary substitutes if available. This option can be - disabled to force building from source. - - - - stalled-download-timeout - - The timeout (in seconds) for receiving data from servers - during download. Nix cancels idle downloads after this timeout's - duration. - - - - substituters - - A list of URLs of substituters, separated by - whitespace. The default is - https://cache.nixos.org. - - - - system - - This option specifies the canonical Nix system - name of the current installation, such as - i686-linux or - x86_64-darwin. Nix can only build derivations - whose system attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only - makes sense if the Nix binaries can run on multiple platforms, - e.g., ‘universal binaries’ that run on x86_64-linux and - i686-linux. - - It defaults to the canonical Nix system name detected by - configure at build time. - - - - - system-features - - A set of system “features” supported by this - machine, e.g. kvm. Derivations can express a - dependency on such features through the derivation attribute - requiredSystemFeatures. For example, the - attribute - - -requiredSystemFeatures = [ "kvm" ]; - - - ensures that the derivation can only be built on a machine with - the kvm feature. - - This setting by default includes kvm if - /dev/kvm is accessible, and the - pseudo-features nixos-test, - benchmark and big-parallel - that are used in Nixpkgs to route builds to specific - machines. - - - - - - tarball-ttl - - - Default: 3600 seconds. - - The number of seconds a downloaded tarball is considered - fresh. If the cached tarball is stale, Nix will check whether - it is still up to date using the ETag header. Nix will download - a new version if the ETag header is unsupported, or the - cached ETag doesn't match. - - - Setting the TTL to 0 forces Nix to always - check if the tarball is up to date. - - Nix caches tarballs in - $XDG_CACHE_HOME/nix/tarballs. - - Files fetched via NIX_PATH, - fetchGit, fetchMercurial, - fetchTarball, and fetchurl - respect this TTL. - - - - - timeout - - - - This option defines the maximum number of seconds that a - builder can run. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite loop - but keep writing to their standard output or standard error. It - can be overridden using the command line - switch. - - The value 0 means that there is no - timeout. This is also the default. - - - - - - trace-function-calls - - - - Default: false. - - If set to true, the Nix evaluator will - trace every function call. Nix will print a log message at the - "vomit" level for every function entrance and function exit. - - -function-trace entered undefined position at 1565795816999559622 -function-trace exited undefined position at 1565795816999581277 -function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 -function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - - The undefined position means the function - call is a builtin. - - Use the contrib/stack-collapse.py script - distributed with the Nix source code to convert the trace logs - in to a format suitable for flamegraph.pl. - - - - - - trusted-public-keys - - A whitespace-separated list of public keys. When - paths are copied from another Nix store (such as a binary cache), - they must be signed with one of these keys. For example: - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=. - - - - trusted-substituters - - A list of URLs of substituters, separated by - whitespace. These are not used by default, but can be enabled by - users of the Nix daemon by specifying --option - substituters urls on the - command line. Unprivileged users are only allowed to pass a - subset of the URLs listed in substituters and - trusted-substituters. - - - - trusted-users - - - - A list of names of users (separated by whitespace) that - have additional rights when connecting to the Nix daemon, such - as the ability to specify additional binary caches, or to import - unsigned NARs. You can also specify groups by prefixing them - with @; for instance, - @wheel means all users in the - wheel group. The default is - root. - - Adding a user to - is essentially equivalent to giving that user root access to the - system. For example, the user can set - and thereby obtain read access to - directories that are otherwise inacessible to - them. - - - - - - - - - - Deprecated Settings - - - - - - - binary-caches - - Deprecated: - binary-caches is now an alias to - . - - - - binary-cache-public-keys - - Deprecated: - binary-cache-public-keys is now an alias to - . - - - - build-compress-log - - Deprecated: - build-compress-log is now an alias to - . - - - - build-cores - - Deprecated: - build-cores is now an alias to - . - - - - build-extra-chroot-dirs - - Deprecated: - build-extra-chroot-dirs is now an alias to - . - - - - build-extra-sandbox-paths - - Deprecated: - build-extra-sandbox-paths is now an alias to - . - - - - build-fallback - - Deprecated: - build-fallback is now an alias to - . - - - - build-max-jobs - - Deprecated: - build-max-jobs is now an alias to - . - - - - build-max-log-size - - Deprecated: - build-max-log-size is now an alias to - . - - - - build-max-silent-time - - Deprecated: - build-max-silent-time is now an alias to - . - - - - build-repeat - - Deprecated: - build-repeat is now an alias to - . - - - - build-timeout - - Deprecated: - build-timeout is now an alias to - . - - - - build-use-chroot - - Deprecated: - build-use-chroot is now an alias to - . - - - - build-use-sandbox - - Deprecated: - build-use-sandbox is now an alias to - . - - - - build-use-substitutes - - Deprecated: - build-use-substitutes is now an alias to - . - - - - gc-keep-derivations - - Deprecated: - gc-keep-derivations is now an alias to - . - - - - gc-keep-outputs - - Deprecated: - gc-keep-outputs is now an alias to - . - - - - env-keep-derivations - - Deprecated: - env-keep-derivations is now an alias to - . - - - - extra-binary-caches - - Deprecated: - extra-binary-caches is now an alias to - . - - - - trusted-binary-caches - - Deprecated: - trusted-binary-caches is now an alias to - . - - - - - - - - From 7f60f48e1aed14680eaf05154abf243ea764bf85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= Date: Wed, 21 Oct 2020 17:04:36 +0200 Subject: [PATCH 053/477] Fix the docs about the new NIX_CONFIG env var This was accidentally documented as NIX_OPTIONS. --- doc/manual/src/command-ref/conf-file-prefix.md | 2 +- doc/manual/src/command-ref/env-common.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index d38456788..e53a3ddf4 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -19,7 +19,7 @@ By default Nix reads settings from the following places: and `XDG_CONFIG_HOME`. If these are unset, it will look in `$HOME/.config/nix.conf`. - - If `NIX_OPTIONS` is set, its contents is treated as the contents of + - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. The configuration files consist of `name = diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 27e730fc8..c670d82b8 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -81,7 +81,7 @@ Most Nix commands interpret the following environment variables: Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). - - `NIX_OPTIONS` + - `NIX_CONFIG` Applies settings from Nix configuration from the environment. The content is treated as if it was read from a Nix configuration file. Settings are separated by the newline character. From e556a1beb73f5d5608d7a8bad0a0c13fd148ed0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Oct 2020 17:54:21 +0200 Subject: [PATCH 054/477] nix develop: Handle 'declare -ax' in bash output Fixes 'nix develop nixpkgs#qpdfview'. --- src/nix/develop.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 380417c82..32669414b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -39,21 +39,24 @@ BuildEnvironment readEnvironment(const Path & path) static std::string varNameRegex = R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; - static std::regex declareRegex( - "^declare -x (" + varNameRegex + ")" + - R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); - static std::string simpleStringRegex = R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; - static std::string quotedStringRegex = + static std::string dquotedStringRegex = + R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + + static std::string squotedStringRegex = R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; static std::string indexedArrayRegex = R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; + static std::regex declareRegex( + "^declare -a?x (" + varNameRegex + ")(=(" + + dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); + static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); /* Note: we distinguish between an indexed and associative array using the space before the closing parenthesis. Will From f9438fb64a223c05ebcfffa9706e1ca811a87d70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 18 Oct 2020 21:06:36 +0200 Subject: [PATCH 055/477] nix develop: Add --redirect flag to redirect dependencies This is primarily useful if you're hacking simultaneously on a package and one of its dependencies. E.g. if you're hacking on Hydra and Nix, you would start a dev shell for Nix, and then a dev shell for Hydra as follows: $ nix develop \ --redirect .#hydraJobs.build.x86_64-linux.nix ~/Dev/nix/outputs/out \ --redirect .#hydraJobs.build.x86_64-linux.nix.dev ~/Dev/nix/outputs/dev (This assumes hydraJobs.build.x86_64-linux has a passthru.nix attribute. You can also use a store path.) This causes all references in the environment to those store paths to be rewritten to ~/Dev/nix/outputs/{out,dev}. Note: unfortunately, you may need to set LD_LIBRARY_PATH=~/Dev/nix/outputs/out/lib because Nixpkgs' ld-wrapper only adds -rpath entries for -L flags that point to the Nix store. --- src/nix/develop.cc | 53 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 32669414b..7a5f7e218 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -185,7 +185,22 @@ struct Common : InstallableCommand, MixProfile "UID", }; + std::vector> redirects; + + Common() + { + addFlag({ + .longName = "redirect", + .description = "redirect a store path to a mutable location", + .labels = {"installable", "outputs-dir"}, + .handler = {[&](std::string installable, std::string outputsDir) { + redirects.push_back({installable, outputsDir}); + }} + }); + } + std::string makeRcScript( + ref store, const BuildEnvironment & buildEnvironment, const Path & outputsDir = absPath(".") + "/outputs") { @@ -217,6 +232,8 @@ struct Common : InstallableCommand, MixProfile out << "eval \"$shellHook\"\n"; + auto script = out.str(); + /* Substitute occurrences of output paths. */ auto outputs = buildEnvironment.env.find("outputs"); assert(outputs != buildEnvironment.env.end()); @@ -230,7 +247,33 @@ struct Common : InstallableCommand, MixProfile rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); } - return rewriteStrings(out.str(), rewrites); + /* Substitute redirects. */ + for (auto & [installableS, dir] : redirects) { + dir = absPath(dir); + auto installable = parseInstallable(store, installableS); + auto buildable = installable->toBuildable(); + auto doRedirect = [&](const StorePath & path) + { + auto from = store->printStorePath(path); + if (script.find(from) == std::string::npos) + warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); + else { + printInfo("redirecting '%s' to '%s'", from, dir); + rewrites.insert({from, dir}); + } + }; + std::visit(overloaded { + [&](const BuildableOpaque & bo) { + doRedirect(bo.path); + }, + [&](const BuildableFromDrv & bfd) { + for (auto & [outputName, path] : bfd.outputs) + if (path) doRedirect(*path); + }, + }, buildable); + } + + return rewriteStrings(script, rewrites); } Strings getDefaultFlakeAttrPaths() override @@ -348,6 +391,10 @@ struct CmdDevelop : Common, MixEnvironment "To use a build environment previously recorded in a profile:", "nix develop /tmp/my-shell" }, + Example{ + "To replace all occurences of a store path with a writable directory:", + "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev" + }, }; } @@ -357,7 +404,7 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(buildEnvironment); + auto script = makeRcScript(store, buildEnvironment); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -449,7 +496,7 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(buildEnvironment); + std::cout << makeRcScript(store, buildEnvironment); } }; From 750ce500c221ecd4720a5b02e3f3cbb0bc05ef9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Oct 2020 12:03:15 +0200 Subject: [PATCH 056/477] Fix clang build --- src/nix/develop.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7a5f7e218..d3c4761a7 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -248,9 +248,9 @@ struct Common : InstallableCommand, MixProfile } /* Substitute redirects. */ - for (auto & [installableS, dir] : redirects) { - dir = absPath(dir); - auto installable = parseInstallable(store, installableS); + for (auto & [installable_, dir_] : redirects) { + auto dir = absPath(dir_); + auto installable = parseInstallable(store, installable_); auto buildable = installable->toBuildable(); auto doRedirect = [&](const StorePath & path) { From c189cf7e3361e3c4caa989bde057439485b2c227 Mon Sep 17 00:00:00 2001 From: tnias Date: Sun, 25 Oct 2020 23:16:53 +0100 Subject: [PATCH 057/477] Add sha512 to hashAlgo listings in manpages (#4186) --- doc/manual/src/command-ref/nix-hash.md | 2 +- doc/manual/src/command-ref/nix-prefetch-url.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 7ed82cdfc..de0459b9e 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -45,7 +45,7 @@ md5sum`. - `--type` *hashAlgo* Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, and `sha256`. + `md5`, `sha1`, `sha256`, and `sha512`. - `--to-base16` Don’t hash anything, but convert the base-32 hash representation diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 78c612cd4..59ab89b29 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,7 +39,7 @@ Nix store is also printed. - `--type` *hashAlgo* Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, and `sha256`. + `md5`, `sha1`, `sha256`, and `sha512`. - `--print-path` Print the store path of the downloaded file on standard output. From ac0e24f21b162f99c8e9437c710797820345a7a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 11:28:42 +0100 Subject: [PATCH 058/477] Revert "Bump version to 3.0" This reverts commit 189e6f5e1d949f50ab0b6e5acd25e230d206692d. After some discussion, it seems better not to bump the major version number since most of the new features since 2.3 are marked experimental. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f398a2061..7208c2182 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.0 \ No newline at end of file +2.4 \ No newline at end of file From dc7d1322efbaa176bff38b1ad15eab6e11c83340 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 14:24:25 +0100 Subject: [PATCH 059/477] Make the prompt used in development shells configurable --- src/nix/develop.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d3c4761a7..8fea7ee9c 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -11,6 +11,19 @@ using namespace nix; +struct DevelopSettings : Config +{ + Setting bashPrompt{this, "", "bash-prompt", + "The bash prompt (`PS1`) in `nix develop` shells."}; + + Setting bashPromptSuffix{this, "", "bash-prompt-suffix", + "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; +}; + +static DevelopSettings developSettings; + +static GlobalConfig::Register rDevelopSettings(&developSettings); + struct Var { bool exported = true; @@ -429,6 +442,10 @@ struct CmdDevelop : Common, MixEnvironment else { script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + if (developSettings.bashPrompt != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); + if (developSettings.bashPromptSuffix != "") + script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix)); } writeFull(rcFileFd.get(), script); From 9d5e9ef0da89fe4fd02d7053ee28d79df3245325 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 16:58:58 +0100 Subject: [PATCH 060/477] Move Explicit --- src/libexpr/flake/lockfile.cc | 3 ++- src/libexpr/primops/fetchTree.cc | 5 +++-- src/libfetchers/attrs.hh | 12 ------------ src/libutil/types.hh | 12 ++++++++++++ src/nix/flake.cc | 6 +++--- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index bb46e1bb4..a01a63611 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -34,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isImmutable()) - throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); + throw Error("lockfile contains mutable lock '%s'", + fetchers::attrsToJson(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 7cd4d0fbf..8d7ae4c14 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -43,7 +43,8 @@ void emitTreeAttrs( } if (input.getType() == "git") - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), + fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); @@ -101,7 +102,7 @@ static void fetchTree( else if (attr.value->type == tString) addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); + attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type == tInt) attrs.emplace(attr.name, attr.value->integer); else diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 4b4630c80..56bcdcfc8 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -8,18 +8,6 @@ namespace nix::fetchers { -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ -template -struct Explicit { - T t; - - bool operator ==(const Explicit & other) const - { - return t == other.t; - } -}; - typedef std::variant> Attr; typedef std::map Attrs; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6c4c5ab74..9c85fef62 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -34,4 +34,16 @@ struct OnStartup OnStartup(T && t) { t(); } }; +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; + + bool operator ==(const Explicit & other) const + { + return t == other.t; + } +}; + } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 43176d887..790e1ce95 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -82,11 +82,11 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake) if (flake.description) j["description"] = *flake.description; j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = attrsToJson(flake.originalRef.toAttrs()); + j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs()); j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); + j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); + j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); if (auto revCount = flake.lockedRef.input.getRevCount()) From 1e66d146a343cff8ea60fa24b1642762de27c787 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 17:59:32 +0100 Subject: [PATCH 061/477] Fix test --- tests/binary-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index e14cf882e..6415c9302 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -271,5 +271,5 @@ rm $cacheDir/$(hashpart $docPath).narinfo nix-store --delete $outPath $docPath # -vvv is the level that logs during the loop -timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ +timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From b875b8f45c8d73c26e2cf13843fa25cc6762eebc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 17:59:36 +0100 Subject: [PATCH 062/477] Remove edition field --- src/libexpr/flake/flake.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bae4d65e5..ca3b185f9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -196,11 +196,6 @@ static Flake getFlake( expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); - auto sEdition = state.symbols.create("edition"); // FIXME: remove soon - - if (vInfo.attrs->get(sEdition)) - warn("flake '%s' has deprecated attribute 'edition'", lockedRef); - if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, tString, *description->value, *description->pos); flake.description = description->value->string.s; @@ -229,8 +224,7 @@ static Flake getFlake( throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); for (auto & attr : *vInfo.attrs) { - if (attr.name != sEdition && - attr.name != state.sDescription && + if (attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs) throw Error("flake '%s' has an unsupported attribute '%s', at %s", From 14aecbb288b71bbb79ddca638918318c9298200b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:36:46 +0100 Subject: [PATCH 063/477] BaseSetting::set(): Don't append to previous value --- src/libutil/config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 521733025..eef01bde2 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -270,6 +270,7 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str) { + value.clear(); auto kvpairs = tokenizeString(str); for (auto & s : kvpairs) { From 731edf0d9be27d1a64c9645595c7efba31dde2b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:37:11 +0100 Subject: [PATCH 064/477] isTrivial(): Support trivial lists --- src/libexpr/eval.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6366050c..4de87d647 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -208,7 +208,8 @@ bool Value::isTrivial() const && (type != tThunk || (dynamic_cast(thunk.expr) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) - || dynamic_cast(thunk.expr)); + || dynamic_cast(thunk.expr) + || dynamic_cast(thunk.expr)); } From 343239fc8a1993f707a990c2cd54a41f1fa3de99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Oct 2020 20:45:39 +0100 Subject: [PATCH 065/477] Allow nix.conf options to be set in flake.nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it possible to have per-project configuration in flake.nix, e.g. binary caches and other stuff: nixConfig.bash-prompt-suffix = "ngi# "; nixConfig.substituters = [ "https://cache.ngi0.nixos.org/" ]; --- src/libexpr/flake/flake.cc | 70 +++++++++++++++++++++++++++++++++++--- src/libexpr/flake/flake.hh | 11 +++++- src/nix/installables.cc | 5 ++- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ca3b185f9..bdcf63c21 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -71,11 +71,17 @@ static std::tuple fetchOrSubstituteTree( return {std::move(tree), resolvedRef, lockedRef}; } -static void expectType(EvalState & state, ValueType type, - Value & value, const Pos & pos) +static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) { if (value.type == tThunk && value.isTrivial()) state.forceValue(value, pos); +} + + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + forceTrivialValue(state, value, pos); if (value.type != type) throw Error("expected %s but got %s at %s", showType(type), showType(value.type), pos); @@ -114,7 +120,6 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, tString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - state.forceValue(*attr.value); if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else @@ -223,10 +228,41 @@ static Flake getFlake( } else throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + auto sNixConfig = state.symbols.create("nixConfig"); + + if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { + expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); + + for (auto & option : *nixConfig->value->attrs) { + forceTrivialValue(state, *option.value, *option.pos); + if (option.value->type == tString) + flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)}); + else if (option.value->type == tInt) + flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)}); + else if (option.value->type == tBool) + flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)}); + else if (option.value->isList()) { + std::vector ss; + for (unsigned int n = 0; n < option.value->listSize(); ++n) { + auto elem = option.value->listElems()[n]; + if (elem->type != tString) + throw TypeError("list element in flake configuration option '%s' is %s while a string is expected", + option.name, showType(*option.value)); + ss.push_back(state.forceStringNoCtx(*elem, *option.pos)); + } + flake.config.options.insert({option.name, ss}); + } + else + throw TypeError("flake configuration option '%s' is %s", + option.name, showType(*option.value)); + } + } + for (auto & attr : *vInfo.attrs) { if (attr.name != state.sDescription && attr.name != sInputs && - attr.name != sOutputs) + attr.name != sOutputs && + attr.name != sNixConfig) throw Error("flake '%s' has an unsupported attribute '%s', at %s", lockedRef, attr.name, *attr.pos); } @@ -599,4 +635,30 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } +void ConfigFile::apply() +{ + for (auto & [name, value] : options) { + // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU. + if (name != "bash-prompt-suffix" && + name != "bash-prompt" && + name != "substituters" && + name != "extra-substituters") + { + warn("ignoring untrusted flake configuration option '%s'", name); + continue; + } + // FIXME: Move into libutil/config.cc. + if (auto s = std::get_if(&value)) + globalConfig.set(name, *s); + else if (auto n = std::get_if(&value)) + globalConfig.set(name, fmt("%d", n)); + else if (auto b = std::get_if>(&value)) + globalConfig.set(name, b->t ? "true" : "false"); + else if (auto ss = std::get_if>(&value)) + globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil + else + assert(false); + } +} + } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index cf62c7741..7eebd9044 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -47,8 +47,16 @@ struct FlakeInput FlakeInputs overrides; }; -// The Flake structure is the main internal representation of a flake.nix file. +struct ConfigFile +{ + using ConfigValue = std::variant, std::vector>; + std::map options; + + void apply(); +}; + +/* The contents of a flake.nix file. */ struct Flake { FlakeRef originalRef; // the original flake specification (by the user) @@ -57,6 +65,7 @@ struct Flake std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; + ConfigFile config; // 'nixConfig' attribute ~Flake(); }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 7473c9758..fb264491a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -533,8 +533,11 @@ InstallableFlake::getCursors(EvalState & state) std::shared_ptr InstallableFlake::getLockedFlake() const { - if (!_lockedFlake) + if (!_lockedFlake) { _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); + _lockedFlake->flake.config.apply(); + // FIXME: send new config to the daemon. + } return _lockedFlake; } From c092fa4702215fdb61611c5dd28194401d056170 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Sep 2020 16:30:42 +0200 Subject: [PATCH 066/477] Allow non-CA derivations to depend on CA derivations --- src/libexpr/primops.cc | 39 +++++++++++----- src/libstore/build/derivation-goal.cc | 12 ++++- src/libstore/derivations.cc | 65 ++++++++++++++++++++++++--- src/libstore/derivations.hh | 14 +++++- src/libstore/local-store.cc | 4 +- src/nix/show-derivation.cc | 1 + tests/content-addressed.nix | 11 ++++- tests/content-addressed.sh | 1 + 8 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..236433ef1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1089,18 +1089,35 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // Regular, non-CA derivation should always return a single hash and not // hash per output. - Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true)); + auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + std::visit(overloaded { + [&](Hash h) { + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }); + } + }, + [&](CaOutputHashes) { + // Shouldn't happen as the toplevel derivation is not CA. + assert(false); + }, + [&](UnknownHashes) { + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputDeferred{}, + }); + } + }, + }, + hashModulo); - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, - }); - } } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index db0c2bb6c..cc8737fd5 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,7 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + if ((!fullDrv.inputDrvs.empty() && + fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3166,6 +3167,15 @@ void DerivationGoal::registerOutputs() [&](DerivationOutputCAFloating dof) { return newInfoFromCA(dof); }, + [&](DerivationOutputDeferred) { + // No derivation should reach that point without having been + // rewritten first + assert(false); + // Ugly, but the compiler insists on having this return a value + // of type `ValidPathInfo` despite the `assert(false)`, so + // let's provide it + return *(ValidPathInfo*)0; + }, }, output.output); /* Calculate where we'll move the output files. In the checking case we diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..3b3a25391 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,6 +21,9 @@ std::optional DerivationOutput::path(const Store & store, std::string [](DerivationOutputCAFloating dof) -> std::optional { return std::nullopt; }, + [](DerivationOutputDeferred) -> std::optional { + return std::nullopt; + }, }, output); } @@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return true; + case DerivationType::DeferredInputAddressed: return false; }; // Since enums can have non-variant values, but making a `default:` would // disable exhaustiveness warnings. @@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store, }; } } else { + if (pathS == "") { + return DerivationOutput { + .output = DerivationOutputDeferred { } + }; + } validatePath(pathS); return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, + [&](DerivationOutputDeferred) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } }, i.second.output); s += ')'; } @@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { @@ -408,22 +424,27 @@ DerivationType BasicDerivation::type() const throw Error("All floating outputs must use the same hash type"); } }, + [&](DerivationOutputDeferred _) { + deferredIAOutputs.insert(i.first); + }, }, i.second.output); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::InputAddressed; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? throw Error("Only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") throw Error("Single fixed output must be named \"out\""); return DerivationType::CAFixed; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::CAFloating; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { + return DerivationType::DeferredInputAddressed; } else { throw Error("Can't mix derivation output types"); } @@ -454,6 +475,8 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } +UnknownHashes unknownHashes; + /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -476,7 +499,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); + return unknownHashes; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -491,12 +514,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } case DerivationType::InputAddressed: break; + case DerivationType::DeferredInputAddressed: + break; } /* For other derivations, replace the inputs paths with recursive calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { + bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation @@ -514,7 +540,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, + [&](UnknownHashes) { + hasUnknownHash = true; + }, }, res); + if (hasUnknownHash) { + return unknownHashes; + } } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); @@ -620,6 +652,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, + [&](DerivationOutputDeferred) { + out << "" + << "" + << ""; + }, }, i.second.output); } worker_proto::write(store, out, drv.inputSrcs); @@ -645,7 +682,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -// N.B. Outputs are left unchanged static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { debug("Rewriting the derivation"); @@ -666,6 +702,21 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String newEnv.emplace(envName, envValue); } drv.env = newEnv; + + auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); + for (auto & [outputName, output] : drv.outputs) { + if (std::holds_alternative(output.output)) { + Hash h = std::get(hashModulo); + auto outPath = store.makeOutputPath(outputName, h, drv.name); + drv.env[outputName] = store.printStorePath(outPath); + output = DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }; + } + } + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d292b2e5..f9ba935e6 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,12 +41,18 @@ struct DerivationOutputCAFloating HashType hashType; }; +/* Input-addressed output which depends on a (CA) derivation whose hash isn't + * known atm + */ +struct DerivationOutputDeferred {}; + struct DerivationOutput { std::variant< DerivationOutputInputAddressed, DerivationOutputCAFixed, - DerivationOutputCAFloating + DerivationOutputCAFloating, + DerivationOutputDeferred > output; std::optional hashAlgoOpt(const Store & store) const; /* Note, when you use this function you should make sure that you're passing @@ -72,6 +78,7 @@ typedef std::map StringPairs; enum struct DerivationType : uint8_t { InputAddressed, + DeferredInputAddressed, CAFixed, CAFloating, }; @@ -167,9 +174,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; +struct UnknownHashes {}; + typedef std::variant< Hash, // regular DRV normalized hash - CaOutputHashes + CaOutputHashes, // Fixed-output derivation hashes + UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29236a9c..d29a68179 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -573,6 +573,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](DerivationOutputCAFloating _) { /* Nothing to check */ }, + [&](DerivationOutputDeferred) { + }, }, i.second.output); } } @@ -817,7 +819,7 @@ std::map> LocalStore::queryPartialDerivati } /* can't just use else-if instead of `!haveCached` because we need to unlock `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && drv.type() == DerivationType::CAFloating) { + if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { /* Try resolve drv and use that path instead. */ auto attempt = drv.tryResolve(*this); if (!attempt) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2542537d3..6d4f295d7 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -82,6 +82,7 @@ struct CmdShowDerivation : InstallablesCommand [&](DerivationOutputCAFloating dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, + [&](DerivationOutputDeferred) {}, }, output.output); } } diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 3dcf916c3..8ca96d4bf 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -15,7 +15,7 @@ rec { ''; }; rootCA = mkDerivation { - name = "dependent"; + name = "rootCA"; outputs = [ "out" "dev" ]; buildCommand = '' echo "building a CA derivation" @@ -51,4 +51,13 @@ rec { outputHashMode = "recursive"; outputHashAlgo = "sha256"; }; + dependentNonCA = mkDerivation { + name = "dependent-non-ca"; + buildCommand = '' + echo "Didn't cut-off" + echo "building dependent-non-ca" + mkdir -p $out + echo ${rootCA}/non-ca-hello > $out/dep + ''; + }; } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 61ec03fe3..547919660 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -22,6 +22,7 @@ secondSeedArgs=(-j0) # dependent derivations always being already built. #testDerivation dependentCA testDerivation transitivelyDependentCA +testDerivation dependentNonCA nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true From bc081bcd816542d66f1578788b93df4d7e07b135 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 24 Sep 2020 10:11:58 +0200 Subject: [PATCH 067/477] Inline `unkownHashes` See https://github.com/NixOS/nix/pull/4056#discussion_r493661632 --- src/libstore/derivations.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3b3a25391..517ecfaa2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -475,8 +475,6 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } -UnknownHashes unknownHashes; - /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -499,7 +497,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - return unknownHashes; + return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -545,7 +543,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m }, }, res); if (hasUnknownHash) { - return unknownHashes; + return UnknownHashes {}; } } From ab21ab65016275c224d1d40c42bdfed80dfbcbb0 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 7 Oct 2020 12:24:53 +0200 Subject: [PATCH 068/477] Test the remote caching of non-ca-depending-on-ca derivations Although the non-resolved derivation will never get a cache-hit (it doesn't have an output path to query the cache for anyways), we might get one on the resolved derivation. --- tests/content-addressed.sh | 55 +++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 547919660..bdab09c86 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -5,24 +5,49 @@ source common.sh drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1) nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 -testDerivation () { +buildAttr () { local derivationPath=$1 - local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + shift + local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +testRemoteCache () { + clearCache + local outPath=$(buildAttr dependentNonCA) + nix copy --to file://$cacheDir $outPath + clearStore + buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") +} + +testDeterministicCA () { + [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]] +} + +testCutoffFor () { local out1 out2 - out1=$(nix-build "${commonArgs[@]}" --arg seed 1) - out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${secondSeedArgs[@]}") + out1=$(buildAttr $1) + # The seed only changes the root derivation, and not it's output, so the + # dependent derivations should only need to be built once. + out2=$(buildAttr $1 -j0) test "$out1" == "$out2" } -testDerivation rootCA -# The seed only changes the root derivation, and not it's output, so the -# dependent derivations should only need to be built once. -secondSeedArgs=(-j0) -# Don't directly build depenentCA, that way we'll make sure we dodn't rely on -# dependent derivations always being already built. -#testDerivation dependentCA -testDerivation transitivelyDependentCA -testDerivation dependentNonCA +testCutoff () { + # Don't directly build depenentCA, that way we'll make sure we dodn't rely on + # dependent derivations always being already built. + #testDerivation dependentCA + testCutoffFor transitivelyDependentCA + testCutoffFor dependentNonCA +} -nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 -nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true +testGC () { + nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 + nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true +} + +testRemoteCache +testDeterministicCA +testCutoff +testGC From 82e4d2a82ef11a95ff8e9aba3062db50145daab1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 28 Oct 2020 05:13:18 +0000 Subject: [PATCH 069/477] No x86_32 static nix jobs for now Fixes #4175 --- flake.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0233962b4..2abbdff53 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,8 @@ officialRelease = false; - linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; + linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; + linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); @@ -228,7 +229,7 @@ # Binary package for various platforms. build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static); + buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -451,7 +452,7 @@ packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; - } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) { + } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { From a5019f0508be961bf0230d2a528d30d3ded4b12a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Oct 2020 20:45:57 +0100 Subject: [PATCH 070/477] Consistency --- src/libutil/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 4a94f0dfd..8efff190a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -192,7 +192,7 @@ Hash Hash::parseAny(std::string_view original, std::optional optType) // Either the string or user must provide the type, if they both do they // must agree. if (!optParsedType && !optType) - throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); else if (optParsedType && optType && *optParsedType != *optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); From 6a4bf535d8126d8dc7306b3940bb49e0cc014a56 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 28 Oct 2020 09:41:18 -0500 Subject: [PATCH 071/477] Capitalize JSON for consistency --- src/libexpr/flake/lockfile.cc | 14 +++++++------- src/libexpr/flake/lockfile.hh | 2 +- src/libfetchers/attrs.cc | 4 ++-- src/libfetchers/attrs.hh | 2 +- src/libfetchers/cache.cc | 20 ++++++++++---------- src/libfetchers/fetchers.cc | 6 +++--- src/libfetchers/registry.cc | 6 +++--- src/libmain/loggers.cc | 4 ++-- src/libmain/loggers.hh | 2 +- src/libutil/logging.cc | 2 +- src/nix/flake.cc | 12 ++++++------ 11 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a01a63611..8e2f7131f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json) { if (!lockedRef.input.isImmutable()) throw Error("lockfile contains mutable lock '%s'", - fetchers::attrsToJson(lockedRef.input.toAttrs())); + fetchers::attrsToJSON(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const @@ -111,7 +111,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) // a bit since we don't need to worry about cycles. } -nlohmann::json LockFile::toJson() const +nlohmann::json LockFile::toJSON() const { nlohmann::json nodes; std::unordered_map, std::string> nodeKeys; @@ -155,8 +155,8 @@ nlohmann::json LockFile::toJson() const } if (auto lockedNode = std::dynamic_pointer_cast(node)) { - n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); - n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); if (!lockedNode->isFlake) n["flake"] = false; } @@ -175,7 +175,7 @@ nlohmann::json LockFile::toJson() const std::string LockFile::to_string() const { - return toJson().dump(2); + return toJSON().dump(2); } LockFile LockFile::read(const Path & path) @@ -186,7 +186,7 @@ LockFile LockFile::read(const Path & path) std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - stream << lockFile.toJson().dump(2); + stream << lockFile.toJSON().dump(2); return stream; } @@ -224,7 +224,7 @@ bool LockFile::isImmutable() const bool LockFile::operator ==(const LockFile & other) const { // FIXME: slow - return toJson() == other.toJson(); + return toJSON() == other.toJSON(); } InputPath parseInputPath(std::string_view s) diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 627794d8c..96f1edc76 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -52,7 +52,7 @@ struct LockFile LockFile() {}; LockFile(const nlohmann::json & json, const Path & path); - nlohmann::json toJson() const; + nlohmann::json toJSON() const; std::string to_string() const; diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 1e59faa73..720b19fcd 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) return attrs; } -nlohmann::json attrsToJson(const Attrs & attrs) +nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { @@ -44,7 +44,7 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin if (i == attrs.end()) return {}; if (auto v = std::get_if(&i->second)) return *v; - throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); + throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } std::string getStrAttr(const Attrs & attrs, const std::string & name) diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 56bcdcfc8..a2d53a7bf 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,7 +13,7 @@ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); -nlohmann::json attrsToJson(const Attrs & attrs); +nlohmann::json attrsToJSON(const Attrs & attrs); std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index e1c7f3dee..34ff6f85b 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -55,8 +55,8 @@ struct CacheImpl : Cache bool immutable) override { _state.lock()->add.use() - (attrsToJson(inAttrs).dump()) - (attrsToJson(infoAttrs).dump()) + (attrsToJSON(inAttrs).dump()) + (attrsToJSON(infoAttrs).dump()) (store->printStorePath(storePath)) (immutable) (time(0)).exec(); @@ -70,7 +70,7 @@ struct CacheImpl : Cache if (!res->expired) return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); debug("ignoring expired cache entry '%s'", - attrsToJson(inAttrs).dump()); + attrsToJSON(inAttrs).dump()); } return {}; } @@ -81,15 +81,15 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto inAttrsJson = attrsToJson(inAttrs).dump(); + auto inAttrsJSON = attrsToJSON(inAttrs).dump(); - auto stmt(state->lookup.use()(inAttrsJson)); + auto stmt(state->lookup.use()(inAttrsJSON)); if (!stmt.next()) { - debug("did not find cache entry for '%s'", inAttrsJson); + debug("did not find cache entry for '%s'", inAttrsJSON); return {}; } - auto infoJson = stmt.getStr(0); + auto infoJSON = stmt.getStr(0); auto storePath = store->parseStorePath(stmt.getStr(1)); auto immutable = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); @@ -97,16 +97,16 @@ struct CacheImpl : Cache store->addTempRoot(storePath); if (!store->isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. - debug("ignoring disappeared cache entry '%s'", inAttrsJson); + debug("ignoring disappeared cache entry '%s'", inAttrsJSON); return {}; } debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJson, infoJson, store->printStorePath(storePath)); + inAttrsJSON, infoJSON, store->printStorePath(storePath)); return Result { .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), - .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), .storePath = std::move(storePath) }; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..e6741a451 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -65,7 +65,7 @@ Input Input::fromAttrs(Attrs && attrs) ParsedURL Input::toURL() const { if (!scheme) - throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs)); return scheme->toURL(*this); } @@ -110,7 +110,7 @@ bool Input::contains(const Input & other) const std::pair Input::fetch(ref store) const { if (!scheme) - throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the @@ -247,7 +247,7 @@ std::optional Input::getLastModified() const ParsedURL InputScheme::toURL(const Input & input) { - throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); + throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } Input InputScheme::applyOverrides( diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 2426882ca..81b2227de 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -60,10 +60,10 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from.toAttrs()); - obj["to"] = attrsToJson(entry.to.toAttrs()); + obj["from"] = attrsToJSON(entry.from.toAttrs()); + obj["to"] = attrsToJSON(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) - obj["to"].update(attrsToJson(entry.extraAttrs)); + obj["to"].update(attrsToJSON(entry.extraAttrs)); if (entry.exact) obj["exact"] = true; arr.emplace_back(std::move(obj)); diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 0a7291780..cdf23859b 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) { else if (logFormatStr == "raw-with-logs") return LogFormat::rawWithLogs; else if (logFormatStr == "internal-json") - return LogFormat::internalJson; + return LogFormat::internalJSON; else if (logFormatStr == "bar") return LogFormat::bar; else if (logFormatStr == "bar-with-logs") @@ -26,7 +26,7 @@ Logger * makeDefaultLogger() { return makeSimpleLogger(false); case LogFormat::rawWithLogs: return makeSimpleLogger(true); - case LogFormat::internalJson: + case LogFormat::internalJSON: return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index cada03110..f3c759193 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -7,7 +7,7 @@ namespace nix { enum class LogFormat { raw, rawWithLogs, - internalJson, + internalJSON, bar, barWithLogs, }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8a6752e22..6fd0dacef 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -306,7 +306,7 @@ bool handleJSONLogMessage(const std::string & msg, } catch (std::exception & e) { logError({ - .name = "Json log message", + .name = "JSON log message", .hint = hintfmt("bad log message from builder: %s", e.what()) }); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 790e1ce95..7a7c71676 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -76,17 +76,17 @@ static void printFlakeInfo(const Store & store, const Flake & flake) std::put_time(std::localtime(&*lastModified), "%F %T")); } -static nlohmann::json flakeToJson(const Store & store, const Flake & flake) +static nlohmann::json flakeToJSON(const Store & store, const Flake & flake) { nlohmann::json j; if (flake.description) j["description"] = *flake.description; j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs()); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs()); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs()); + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -139,7 +139,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON auto flake = getFlake(); if (json) { - auto json = flakeToJson(*store, flake); + auto json = flakeToJSON(*store, flake); logger->cout("%s", json.dump()); } else printFlakeInfo(*store, flake); @@ -158,7 +158,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON auto flake = lockFlake(); if (json) - logger->cout("%s", flake.lockFile.toJson()); + logger->cout("%s", flake.lockFile.toJSON()); else { logger->cout("%s", flake.flake.lockedRef); From 869c0321ff4829f56e10316f401a0f9268c1a8b0 Mon Sep 17 00:00:00 2001 From: stev Date: Thu, 29 Oct 2020 00:33:14 +0100 Subject: [PATCH 072/477] Alter "wanted:" to "specified:" in hash mismatch output This makes it even clearer which of the two hashes was specified in the nix files. Some may think that "wanted" and "got" is obvious, but: "got" could mean "got in nix file" and "wanted" could mean "want to see in nix file". --- src/libexpr/primops/fetchTree.cc | 2 +- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/local-store.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8d7ae4c14..e6f637a43 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? state.store->queryPathInfo(storePath)->narHash : hashFile(htSHA256, path); if (hash != *expectedHash) - throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index cc8737fd5..7db83c8be 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3157,7 +3157,7 @@ void DerivationGoal::registerOutputs() valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), wanted.to_string(SRI, true), got.to_string(SRI, true))); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29a68179..bfad8fb21 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1085,11 +1085,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); autoGC(); From ff4dea63c9403880500f82ce273713ecf793d2d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Oct 2020 18:17:39 +0100 Subject: [PATCH 073/477] Generalize extra-* settings This removes the extra-substituters and extra-sandbox-paths settings and instead makes every array setting extensible by setting "extra- = " in the configuration file or passing "-- " on the command line. --- .../src/command-ref/conf-file-prefix.md | 27 ++++++--- src/libstore/build/derivation-goal.cc | 6 +- src/libstore/daemon.cc | 2 - src/libstore/globals.cc | 9 ++- src/libstore/globals.hh | 21 +------ src/libstore/store-api.cc | 3 - src/libutil/config.cc | 58 +++++++++++++++---- src/libutil/config.hh | 11 +++- 8 files changed, 83 insertions(+), 54 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index e53a3ddf4..3140170ab 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -22,19 +22,30 @@ By default Nix reads settings from the following places: - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. -The configuration files consist of `name = -value` pairs, one per line. Other files can be included with a line like -`include -path`, where *path* is interpreted relative to the current conf file and -a missing file is an error unless `!include` is used instead. Comments +The configuration files consist of `name = value` pairs, one per +line. Other files can be included with a line like `include path`, +where *path* is interpreted relative to the current conf file and a +missing file is an error unless `!include` is used instead. Comments start with a `#` character. Here is an example configuration file: keep-outputs = true # Nice for developers keep-derivations = true # Idem -You can override settings on the command line using the `--option` flag, -e.g. `--option keep-outputs -false`. +You can override settings on the command line using the `--option` +flag, e.g. `--option keep-outputs false`. Every configuration setting +also has a corresponding command line flag, e.g. `--max-jobs 16`; for +Boolean settings, there are two flags to enable or disable the setting +(e.g. `--keep-failed` and `--no-keep-failed`). + +A configuration setting usually overrides any previous value. However, +you can prefix the name of the setting by `extra-` to *append* to the +previous value. For instance, + + substituters = a b + extra-substituters = c d + +defines the `substituters` setting to be `a b c d`. This is also +available as a command line flag (e.g. `--extra-substituters`). The following settings are currently available: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7db83c8be..3dacb218c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1332,13 +1332,9 @@ void DerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - dirsInChroot.clear(); - for (auto i : dirs) { + for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; bool optional = false; if (i[i.size() - 1] == '?') { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 99d8add92..4dbc7ba38 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -231,8 +231,6 @@ struct ClientSettings settings.set(name, value); else if (setSubstituters(settings.substituters)) ; - else if (setSubstituters(settings.extraSubstituters)) - ; else debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4df68d0c9..f38601d6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -160,7 +160,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { if (str == "true") value = smEnabled; else if (str == "relaxed") value = smRelaxed; @@ -168,6 +168,11 @@ template<> void BaseSetting::set(const std::string & str) else throw UsageError("option '%s' has invalid value '%s'", name, str); } +template<> bool BaseSetting::isAppendable() +{ + return false; +} + template<> std::string BaseSetting::to_string() const { if (value == smEnabled) return "true"; @@ -198,7 +203,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str) +void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); else if (!string2Int(str, value)) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8c63c5b34..eabd83e3f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -25,7 +25,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; }; class Settings : public Config { @@ -413,14 +413,6 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting extraSandboxPaths{ - this, {}, "extra-sandbox-paths", - R"( - A list of additional paths appended to `sandbox-paths`. Useful if - you want to extend its default value. - )", - {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting buildRepeat{ this, 0, "repeat", R"( @@ -599,17 +591,6 @@ public: )", {"binary-caches"}}; - // FIXME: provide a way to add to option values. - Setting extraSubstituters{ - this, {}, "extra-substituters", - R"( - Additional binary caches appended to those specified in - `substituters`. When used by unprivileged users, untrusted - substituters (i.e. those not listed in `trusted-substituters`) are - silently ignored. - )", - {"extra-binary-caches"}}; - Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9f21f0434..83d3a1fa1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1114,9 +1114,6 @@ std::list> getDefaultSubstituters() for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); - stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index eef01bde2..116dd6bfe 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -8,9 +8,18 @@ namespace nix { bool Config::set(const std::string & name, const std::string & value) { + bool append = false; auto i = _settings.find(name); - if (i == _settings.end()) return false; - i->second.setting->set(value); + if (i == _settings.end()) { + if (hasPrefix(name, "extra-")) { + i = _settings.find(std::string(name, 6)); + if (i == _settings.end() || !i->second.setting->isAppendable()) + return false; + append = true; + } else + return false; + } + i->second.setting->set(value, append); i->second.setting->overriden = true; return true; } @@ -180,6 +189,12 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } +template +bool BaseSetting::isAppendable() +{ + return false; +} + template void BaseSetting::convertToArg(Args & args, const std::string & category) { @@ -190,9 +205,18 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s); }}, }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s, true); }}, + }); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { value = str; } @@ -203,7 +227,7 @@ template<> std::string BaseSetting::to_string() const } template -void BaseSetting::set(const std::string & str) +void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); if (!string2Int(str, value)) @@ -217,7 +241,7 @@ std::string BaseSetting::to_string() const return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { if (str == "true" || str == "yes" || str == "1") value = true; @@ -248,9 +272,16 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { - value = tokenizeString(str); + auto ss = tokenizeString(str); + if (!append) value.clear(); + for (auto & s : ss) value.push_back(std::move(s)); +} + +template<> bool BaseSetting::isAppendable() +{ + return true; } template<> std::string BaseSetting::to_string() const @@ -258,7 +289,7 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { value = tokenizeString(str); } @@ -268,9 +299,9 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { - value.clear(); + if (!append) value.clear(); auto kvpairs = tokenizeString(str); for (auto & s : kvpairs) { @@ -281,6 +312,11 @@ template<> void BaseSetting::set(const std::string & str) } } +template<> bool BaseSetting::isAppendable() +{ + return true; +} + template<> std::string BaseSetting::to_string() const { Strings kvstrs; @@ -301,7 +337,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; -void PathSetting::set(const std::string & str) +void PathSetting::set(const std::string & str, bool append) { if (str == "") { if (allowEmpty) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1f5f4e7b9..71e31656d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -202,7 +202,10 @@ protected: assert(created == 123); } - virtual void set(const std::string & value) = 0; + virtual void set(const std::string & value, bool append = false) = 0; + + virtual bool isAppendable() + { return false; } virtual std::string to_string() const = 0; @@ -243,7 +246,9 @@ public: void operator =(const T & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; + + bool isAppendable() override; virtual void override(const T & v) { @@ -305,7 +310,7 @@ public: options->addSetting(this); } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; Path operator +(const char * p) const { return value + p; } From 7f56cf67bac3731ed8e217170eb548bf0fd2cfcb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Oct 2020 18:26:35 +0100 Subject: [PATCH 074/477] Fix assertion failure in tab completion for --option --- src/libmain/common-args.cc | 2 +- src/libutil/config.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9151a0344..3e4e475e5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, s.second.description); + completions->add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 116dd6bfe..be957dfe3 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -200,7 +200,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, - .description = description, + .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s); }}, @@ -209,7 +209,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) if (isAppendable()) args.addFlag({ .longName = "extra-" + name, - .description = description, + .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, .handler = {[=](std::string s) { overriden = true; set(s, true); }}, @@ -260,13 +260,13 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, - .description = description, + .description = fmt("Enable the `%s` setting.", name), .category = category, .handler = {[=]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, - .description = description, + .description = fmt("Disable the `%s` setting.", name), .category = category, .handler = {[=]() { override(false); }} }); From b809c48ebba7b628a5fb0a0e284cf7068589d479 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 11:01:33 +0100 Subject: [PATCH 075/477] nix-shell.md: evaluated -> run Use "run" to avoid confusion with Nix evaluation. "evaluated" was intended to reference bash eval but it's ambiguous. --- doc/manual/src/command-ref/nix-shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index d1266930e..8c77923d0 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -32,7 +32,7 @@ URL of a tarball that will be downloaded and unpacked to a temporary location. The tarball must include a single top-level directory containing at least a file named `default.nix`. -If the derivation defines the variable `shellHook`, it will be evaluated +If the derivation defines the variable `shellHook`, it will be run after `$stdenv/setup` has been sourced. Since this hook is not executed by regular Nix builds, it allows you to perform initialisation specific to `nix-shell`. For example, the derivation attribute From d4c5d8d32a4bc5b3eaa63add23dca9d755dc5a74 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 11:12:28 +0100 Subject: [PATCH 076/477] nix-shell.md: Extend shellHook example --- doc/manual/src/command-ref/nix-shell.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 8c77923d0..88b675e71 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -41,10 +41,12 @@ to `nix-shell`. For example, the derivation attribute shellHook = '' echo "Hello shell" + export SOME_API_TOKEN="$(cat ~/.config/some-app/api-token)" ''; ``` -will cause `nix-shell` to print `Hello shell`. +will cause `nix-shell` to print `Hello shell` and set the `SOME_API_TOKEN` +environment variable to a user-configured value. # Options From dc5696b84f55a6706cddc3d747ef1aeffb564f43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Oct 2020 12:00:43 +0100 Subject: [PATCH 077/477] Fix test --- src/libutil/tests/config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c7777a21f..c305af9f5 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -80,8 +80,8 @@ namespace nix { class TestSetting : public AbstractSetting { public: TestSetting() : AbstractSetting("test", "test", {}) {} - void set(const std::string & value) {} - std::string to_string() const { return {}; } + void set(const std::string & value, bool append) override {} + std::string to_string() const override { return {}; } }; Config config; From c4d903ddb009aa6472530699e154d85a24eac51d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 20:55:53 +0100 Subject: [PATCH 078/477] Fix memory corruption caused by GC-invisible coroutine stacks Crucially this introduces BoehmGCStackAllocator, but it also adds a bunch of wiring to avoid making libutil depend on bdw-gc. Part of the solutions for #4178, #4200 --- src/libexpr/eval.cc | 26 ++++++++++++++++++++++++++ src/libexpr/local.mk | 2 +- src/libutil/serialise.cc | 35 ++++++++++++++++++++++++++++++++++- src/libutil/serialise.hh | 14 ++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4de87d647..ea7ba0a6a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -27,6 +27,10 @@ #include #include +#include +#include +#include + #endif namespace nix { @@ -220,6 +224,26 @@ static void * oomHandler(size_t requested) /* Convert this to a proper C++ exception. */ throw std::bad_alloc(); } + +class BoehmGCStackAllocator : public StackAllocator { + boost::coroutines2::protected_fixedsize_stack stack; + + public: + boost::context::stack_context allocate() override { + auto sctx = stack.allocate(); + GC_add_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + return sctx; + } + + void deallocate(boost::context::stack_context sctx) override { + GC_remove_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + stack.deallocate(sctx); + } + +}; + +static BoehmGCStackAllocator boehmGCStackAllocator; + #endif @@ -257,6 +281,8 @@ void initGC() GC_set_oom_fn(oomHandler); + StackAllocator::defaultAllocator = &boehmGCStackAllocator; + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 687a8ccda..a5422169d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS = +libexpr_LDFLAGS = -lboost_context ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5c9f6f901..28f6968d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,6 +171,39 @@ size_t StringSource::read(unsigned char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif +/* A concrete datatype allow virtual dispatch of stack allocation methods. */ +struct VirtualStackAllocator { + StackAllocator *allocator = StackAllocator::defaultAllocator; + + boost::context::stack_context allocate() { + return allocator->allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + allocator->deallocate(sctx); + } +}; + + +/* This class reifies the default boost coroutine stack allocation strategy with + a virtual interface. */ +class DefaultStackAllocator : public StackAllocator { + boost::coroutines2::default_stack stack; + + boost::context::stack_context allocate() { + return stack.allocate(); + } + + void deallocate(boost::context::stack_context sctx) { + deallocate(sctx); + } +}; + +static DefaultStackAllocator defaultAllocatorSingleton; + +StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; + + std::unique_ptr sinkToSource( std::function fun, std::function eof) @@ -195,7 +228,7 @@ std::unique_ptr sinkToSource( size_t read(unsigned char * data, size_t len) override { if (!coro) - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](const unsigned char * data, size_t len) { if (len) yield(std::string((const char *) data, len)); }); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d7fe0b81e..5c7d3ce76 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +namespace boost::context { struct stack_context; } namespace nix { @@ -497,5 +498,18 @@ struct FramedSink : nix::BufferedSink }; }; +/* Stack allocation strategy for sinkToSource. + Mutable to avoid a boehm gc dependency in libutil. + + boost::context doesn't provide a virtual class, so we define our own. + */ +struct StackAllocator { + virtual boost::context::stack_context allocate() = 0; + virtual void deallocate(boost::context::stack_context sctx) = 0; + + /* The stack allocator to use in sinkToSource and potentially elsewhere. + It is reassigned by the initGC() method in libexpr. */ + static StackAllocator *defaultAllocator; +}; } From 2192cac634dcd13c3365f4dfeb5f393f4fd9e327 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 21:47:34 +0100 Subject: [PATCH 079/477] Fix RemoteStore pool deadlock in filterSource etc --- src/libstore/remote-store.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..ccbd3bb1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -471,9 +471,14 @@ ref RemoteStore::addCAToStore( worker_proto::write(*this, conn->to, references); conn->to << repair; - conn.withFramedSink([&](Sink & sink) { - dump.drainInto(sink); - }); + // The dump source may invoke the store, so we need to make some room. + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + } auto path = parseStorePath(readString(conn->from)); return readValidPathInfo(conn, path); From b43c13a9161daf1801188e61104debafa5243fe1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 23:18:24 +0100 Subject: [PATCH 080/477] BoehmGCStackAllocator: increase stack size to 8MB The default stack size was not based on the normal stack size and was too small. --- src/libexpr/eval.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ea7ba0a6a..486a9fc1a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -226,7 +226,12 @@ static void * oomHandler(size_t requested) } class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack; + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) + }; public: boost::context::stack_context allocate() override { From e8a45d07bccc315de88b8434393082eee6e944a8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 31 Oct 2020 23:52:09 +0100 Subject: [PATCH 081/477] Restore RestrictedStore.addToStoreFromDump implementation It was accidentally removed in commit ca30abb3fb36440e5a13161c39647189036fc18f --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3dacb218c..19d96dd8f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2074,6 +2074,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf return path; } + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + void narFromPath(const StorePath & path, Sink & sink) override { if (!goal.isAllowed(path)) From db5424bf09886afc1c81db36766522f68fc66ba8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 13:57:58 +0100 Subject: [PATCH 082/477] Don't send eval-related settings to the daemon --- src/libstore/remote-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..fb52ca6d0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -11,6 +11,7 @@ #include "finally.hh" #include "logging.hh" #include "callback.hh" +#include "filetransfer.hh" namespace nix { @@ -171,7 +172,8 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map overrides; - globalConfig.getSettings(overrides, true); + settings.getSettings(overrides, true); // libstore settings + fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); overrides.erase(settings.keepGoing.name); overrides.erase(settings.tryFallback.name); From 8b15650e7421cf9433f04c32ac73601aada1e3ab Mon Sep 17 00:00:00 2001 From: mkenigs <40775676+mkenigs@users.noreply.github.com> Date: Mon, 2 Nov 2020 09:32:05 -0600 Subject: [PATCH 083/477] docs: consistent console prompt (#4213) Everywhere else a $ is used --- doc/manual/src/installation/installing-binary.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 8b8d1d738..ae7fd458b 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -195,7 +195,7 @@ If you are comfortable navigating these tradeoffs, you can encrypt the volume with something along the lines of: ```console -alice$ diskutil apfs enableFileVault /nix -user disk +$ diskutil apfs enableFileVault /nix -user disk ``` ## Symlink the Nix store to a custom location @@ -234,13 +234,13 @@ as a helpful reference if you run into trouble. without a reboot: ```console - alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B ``` 3. Create the new APFS volume with diskutil: ```console - alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix + $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix ``` 4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't @@ -280,10 +280,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install` inside the binary tarball: ```console -alice$ cd /tmp -alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -alice$ cd nix-1.8-x86_64-darwin -alice$ ./install +$ cd /tmp +$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +$ cd nix-1.8-x86_64-darwin +$ ./install ``` If you need to edit the multi-user installation script to use different From 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 18:46:44 +0100 Subject: [PATCH 084/477] Don't use readDerivation() in addValidPath() readDerivation() requires a valid path. Fixes #4210. --- src/libstore/local-store.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bfad8fb21..2892b0407 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -623,7 +623,10 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(info.path); + auto drv = parseDerivation( + *this, + readFile(Store::toRealPath(info.path)), + Derivation::nameFromPath(info.path)); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1000,7 +1003,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, readDerivation(i.path)); + checkDerivationOutputs(i.path, + parseDerivation( + *this, + readFile(Store::toRealPath(i.path)), + Derivation::nameFromPath(i.path))); } /* Do a topological sort of the paths. This will throw an From 550e11f077ae508abde5a33998a9d4029880e7b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 19:07:37 +0100 Subject: [PATCH 085/477] nix repl: Fix handling of multi-line expressions --- src/nix/repl.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9ff386b1d..71794a309 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -212,7 +212,7 @@ void NixRepl::mainLoop(const std::vector & files) try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { - if (e.msg().find("unexpected $end") != std::string::npos) { + if (e.msg().find("unexpected end of file") != std::string::npos) { // For parse errors on incomplete input, we continue waiting for the next line of // input without clearing the input so far. continue; @@ -220,9 +220,9 @@ void NixRepl::mainLoop(const std::vector & files) printMsg(lvlError, e.msg()); } } catch (Error & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Interrupted & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } // We handled the current input fully, so we should clear it From 8cd2ff69c38913d5fcddeb0a012ba0cf34de0686 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 2 Nov 2020 15:50:14 -0500 Subject: [PATCH 086/477] nix-copy-closure: verify it works with drvs Creates test coverage for #4210 and 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 --- tests/nix-copy-closure.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index e5f6a0f12..1b63a3fca 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -7,14 +7,14 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { name = "nix-copy-closure"; nodes = { client = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgA ]; + virtualisation.pathsInNixDB = [ pkgA pkgD.drvPath ]; nix.binaryCaches = lib.mkForce [ ]; }; @@ -60,6 +60,12 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { # Copy the closure of package C via the SSH substituter. client.fail("nix-store -r ${pkgC}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix-copy-closure --to server --gzip ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + # FIXME # client.succeed( # "nix-store --option use-ssh-substituter true" From 797a52e31d97381e0be7e0a80e43f8cb259f8a6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:26:53 +0100 Subject: [PATCH 087/477] Add FIXME --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f1e2ab7f9..5e07261a6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,6 +90,8 @@ private: std::unique_ptr publicKeys; }; + // FIXME: get rid of recursive_mutex, it hides recursive SQLite + // queries. Sync _state; public: From e8c379555fa0441c5ab83b8e5a3a106d69fe2507 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:52:57 +0100 Subject: [PATCH 088/477] LocalStore: Get rid of recursive_mutex --- src/libstore/build/derivation-goal.cc | 6 +++--- src/libstore/local-store.cc | 23 ++++++++++++++++------- src/libstore/local-store.hh | 4 +--- src/libstore/path-info.hh | 2 +- src/nix-store/nix-store.cc | 2 +- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 19d96dd8f..bf2bad62c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3245,7 +3245,7 @@ void DerivationGoal::registerOutputs() if (!oldInfo.ultimate) { oldInfo.ultimate = true; worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); + worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3275,7 +3275,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); + worker.store.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3350,7 +3350,7 @@ void DerivationGoal::registerOutputs() { ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); + infos2.insert_or_assign(newInfo.path, newInfo); } worker.store.registerValidPaths(infos2); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2892b0407..e4e404dca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -7,6 +7,7 @@ #include "nar-info.hh" #include "references.hh" #include "callback.hh" +#include "topo-sort.hh" #include #include @@ -962,9 +963,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst void LocalStore::registerValidPath(const ValidPathInfo & info) { - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); + registerValidPaths({{info.path, info}}); } @@ -982,7 +981,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) SQLiteTxn txn(state->db); StorePathSet paths; - for (auto & i : infos) { + for (auto & [_, i] : infos) { assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); @@ -991,7 +990,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i.path); } - for (auto & i : infos) { + for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); @@ -1000,7 +999,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Check that the derivation outputs are correct. We can't do this in addValidPath() above, because the references might not be valid yet. */ - for (auto & i : infos) + for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, @@ -1014,7 +1013,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(paths); + topoSort(paths, + {[&](const StorePath & path) { + auto i = infos.find(path); + return i == infos.end() ? StorePathSet() : i->second.references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); txn.commit(); }); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5e07261a6..d4435220d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,9 +90,7 @@ private: std::unique_ptr publicKeys; }; - // FIXME: get rid of recursive_mutex, it hides recursive SQLite - // queries. - Sync _state; + Sync _state; public: diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 8ff5c466e..de87f8b33 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -107,6 +107,6 @@ struct ValidPathInfo virtual ~ValidPathInfo() { } }; -typedef list ValidPathInfos; +typedef std::map ValidPathInfos; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 14baabc36..6c2702bfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -516,7 +516,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) info->narHash = hash.first; info->narSize = hash.second; } - infos.push_back(std::move(*info)); + infos.insert_or_assign(info->path, *info); } } From 5e6eabe1551f3fa2a463fced39a4a48b504ed1ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 14:44:52 +0100 Subject: [PATCH 089/477] Fix error message 'assertion failed at' --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4de87d647..cb8e0a0de 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1405,7 +1405,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); + throwAssertionError(pos, "assertion '%1%' failed", out.str()); } body->eval(state, env, v); } From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 21 Oct 2020 21:31:19 +0200 Subject: [PATCH 090/477] Allow substituting paths when building remotely using `ssh-ng://` Until now, it was not possible to substitute missing paths from e.g. `https://cache.nixos.org` on a remote server when building on it using the new `ssh-ng` protocol. This is because every store implementation except legacy `ssh://` ignores the substitution flag passed to `Store::queryValidPaths` while the `legacy-ssh-store` substitutes the remote store using `cmdQueryValidPaths` when the remote store is opened with `nix-store --serve`. This patch slightly modifies the daemon protocol to allow passing an integer value suggesting whether to substitute missing paths during `wopQueryValidPaths`. To implement this on the daemon-side, the substitution logic from `nix-store --serve` has been moved into a protected method named `Store::substitutePaths` which gets currently called from `LocalStore::queryValidPaths` and `Store::queryValidPaths` if `maybeSubstitute` is `true`. Fixes #2770 --- flake.nix | 3 ++- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/remote-store.cc | 3 +++ src/libstore/store-api.cc | 22 ++++++++++++++++++++++ src/libstore/store-api.hh | 5 +++++ src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 23 +---------------------- 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/flake.nix b/flake.nix index 2abbdff53..8ff048be3 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ versionSuffix = if officialRelease then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}"; + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; officialRelease = false; @@ -117,6 +117,7 @@ nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { name = "nix-${version}"; + inherit version; src = self; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4dbc7ba38..60cca4fda 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryValidPaths: { auto paths = worker_proto::read(*store, from, Phantom {}); + + SubstituteFlag substitute = NoSubstitute; + if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { + substitute = readInt(from) ? Substitute : NoSubstitute; + } + logger->startWork(); - auto res = store->queryValidPaths(paths); + if (substitute) { + store->substitutePaths(paths); + } + auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); worker_proto::write(*store, to, res); break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..c27a0f278 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute } else { conn->to << wopQueryValidPaths; worker_proto::write(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { + conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + } conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..3129d6637 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath, } +void Store::substitutePaths(const StorePathSet & paths) +{ + std::vector paths2; + for (auto & path : paths) + if (!path.isDerivation()) + paths2.push_back({path}); + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + queryMissing(paths2, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willSubstitute.empty()) + try { + std::vector subs; + for (auto & p : willSubstitute) subs.push_back({p}); + buildPaths(subs); + } catch (Error & e) { + logWarning(e.info()); + } +} + + StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..9a373b561 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,6 +360,11 @@ protected: public: + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + void substitutePaths(const StorePathSet & paths); + /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b3705578e..63bd6ea49 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11a +#define PROTOCOL_VERSION 0x11b #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6c2702bfe..54394e921 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) store->addTempRoot(path); - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ if (substitute && writeAllowed) { - /* Filter out .drv files (we don't want to build anything). */ - std::vector paths2; - for (auto & path : paths) - if (!path.isDerivation()) - paths2.push_back({path}); - uint64_t downloadSize, narSize; - StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths2, - willBuild, willSubstitute, unknown, downloadSize, narSize); - /* FIXME: should use ensurePath(), but it only - does one path at a time. */ - if (!willSubstitute.empty()) - try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); - store->buildPaths(subs); - } catch (Error & e) { - logWarning(e.info()); - } + store->substitutePaths(paths); } worker_proto::write(*store, out, store->queryValidPaths(paths)); From b87f84cf55b4ca666b31c511e2489789e3322da4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 15:04:34 +0100 Subject: [PATCH 091/477] Fix appending to Setting Fixes: warning: unknown setting 'extra-sandbox-paths' --- src/libutil/config.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index be957dfe3..7af3e7883 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -291,7 +291,14 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str, bool append) { - value = tokenizeString(str); + if (!append) value.clear(); + for (auto & s : tokenizeString(str)) + value.insert(s); +} + +template<> bool BaseSetting::isAppendable() +{ + return true; } template<> std::string BaseSetting::to_string() const @@ -302,9 +309,7 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::set(const std::string & str, bool append) { if (!append) value.clear(); - auto kvpairs = tokenizeString(str); - for (auto & s : kvpairs) - { + for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); From fb7735e4cf3a1ee6337bf1f2ee15204bb11304b2 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Sat, 7 Nov 2020 15:00:22 +0100 Subject: [PATCH 092/477] nix develop: Preserve stdin with `-c` --- src/nix/develop.cc | 4 ++-- tests/nix-shell.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8fea7ee9c..457d94382 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -474,9 +474,9 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } - // If running a phase, don't want an interactive shell running after + // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile - auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath} + auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 1228bb04f..dfe8ed0c1 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,6 +58,7 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' +echo foo | nix develop -f shell.nix shellDrv -c cat # preserve stdin with `-c` # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) From 579b953231599f111dcc9fac95491458c9f6ef08 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Mon, 9 Nov 2020 17:50:51 +0100 Subject: [PATCH 093/477] Make test case more precise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt --- tests/nix-shell.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index dfe8ed0c1..3b000a049 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,7 +58,8 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' -echo foo | nix develop -f shell.nix shellDrv -c cat # preserve stdin with `-c` +# Preserve stdin with `-c` +echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) From 3f24a417dacd5a5964f781f4d704a5253401d380 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Mon, 9 Nov 2020 17:57:39 +0100 Subject: [PATCH 094/477] Add test case for incidentally fixed #4228 --- tests/nix-shell.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 3b000a049..7b2be650a 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -58,9 +58,13 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' -# Preserve stdin with `-c` + +# Ensure `nix develop -c` preserves stdin echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo +# Ensure `nix develop -c` actually executes the command if stdout isn't a terminal +nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo + # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) [[ -n $stdenv ]] From 0ed7c957bed18d963df65d55f0c0ffc79dee656d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 23:21:55 +0000 Subject: [PATCH 095/477] Bump cachix/install-nix-action from v11 to v12 (#4237) Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v11 to v12. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v11...07da2520eebede906fbeefa9dd0a2b635323909d) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 829111b67..021642f4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v11 + - uses: cachix/install-nix-action@v12 #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) From 107c91f5fe6248548c292d28d0ad53c0de7ceeba Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 16:48:35 -0700 Subject: [PATCH 096/477] auto-call error --- src/libexpr/eval.cc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6366050c..e52e8dcf2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,7 +1370,28 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + An 'auto-call' is when a nix expression is evaluated without any external arguments. If that + nix expression is a function, and that function's arguments all have default values, then all is well. + + But if the function arguments don't have default values, then evaluation fails. + + The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... + + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + + See this nix pill for more information re callPackage format: + https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 6c2933a8d72f9328a2931a8166439bed96b80f24 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:04:52 -0700 Subject: [PATCH 097/477] add position --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e52e8dcf2..11f2e31ce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,11 +1370,11 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default values, then all is well. - But if the function arguments don't have default values, then evaluation fails. + But if the function arguments don't have default values, evaluation fails. The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. From 9f2b25ce55e38fc5772fcb65bd98651255f6a49b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:17:47 -0700 Subject: [PATCH 098/477] remove unused ftn; reformat line breaks --- src/libexpr/eval.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 11f2e31ce..0446376f3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -636,11 +636,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(s, s1); -} - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ @@ -1371,12 +1366,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. If that - nix expression is a function, and that function's arguments all have default values, then all is well. + An 'auto-call' is when a nix expression is evaluated without any external arguments. + If that nix expression is a function, and that function's arguments all have default + values, then all is well. But if the function arguments don't have default values, evaluation fails. - The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + The classic case for this error is evaluating a nix file with nix-build that expects + to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. { stdenv }: From d8ef423a189b95f9f6e0a385a0f4474bebf297ef Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 19:16:50 -0700 Subject: [PATCH 099/477] error message formatting --- src/libexpr/eval.cc | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0446376f3..2b697882b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1366,29 +1366,30 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. - If that nix expression is a function, and that function's arguments all have default - values, then all is well. - But if the function arguments don't have default values, evaluation fails. +An 'auto-call' is when a nix expression is evaluated without any external arguments. +If that nix expression is a function, and that function's arguments all have default +values, then all is well. - The classic case for this error is evaluating a nix file with nix-build that expects - to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... +But if the function arguments don't have default values, evaluation fails. - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... +The classic case for this error is evaluating a nix file with nix-build that expects +to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... - See this nix pill for more information re callPackage format: - https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + +More about callPackage: +https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 108a2dab7e460533064b24f5dff05adc453acb27 Mon Sep 17 00:00:00 2001 From: "Ricardo M. Correia" Date: Tue, 10 Nov 2020 04:24:55 +0100 Subject: [PATCH 100/477] Fix stack overflow introduced in #4206 --- src/libutil/serialise.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 28f6968d0..038ede049 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -195,7 +195,7 @@ class DefaultStackAllocator : public StackAllocator { } void deallocate(boost::context::stack_context sctx) { - deallocate(sctx); + stack.deallocate(sctx); } }; From 4864df6d6b7f5fc5c0815bf7733c40663ea856d7 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 10 Nov 2020 08:48:49 -0600 Subject: [PATCH 101/477] enable Darwin.arm64 to install x86_64 binary Throwing @thefloweringash under the bus if this doesn't work, but it sounds like Apple Silicon devices can use the x86_64 binary for now. Fixes #4058 --- scripts/install.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install.in b/scripts/install.in index 39fae37e3..9a281d776 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -29,6 +29,8 @@ case "$(uname -s).$(uname -m)" in Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;; + Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From 4badb6943f73e3d740d4de793eb9860e26842bc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Nov 2020 23:22:45 +0100 Subject: [PATCH 102/477] Fix use of dirty Git/Mercurial inputs with chroot stores Fixes: $ nix build --store /tmp/nix /home/eelco/Dev/patchelf#hydraJobs.build.x86_64-linux warning: Git tree '/home/eelco/Dev/patchelf' is dirty error: --- RestrictedPathError ------------------------------------------------------------------------------------------- nix access to path '/tmp/nix/nix/store/xmkvfmffk7xfnazykb5kx999aika8an4-source/flake.nix' is forbidden in restricted mode (use '--show-trace' to show detailed location information) --- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a6411b02b..e7712c5fd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -273,7 +273,7 @@ struct GitInputScheme : InputScheme haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return { - Tree(store->printStorePath(storePath), std::move(storePath)), + Tree(store->toRealPath(storePath), std::move(storePath)), input }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 7d3d52751..41f60c45c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -166,7 +166,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return { - Tree(store->printStorePath(storePath), std::move(storePath)), + Tree(store->toRealPath(storePath), std::move(storePath)), input }; } From 7d9037035ef8bfe0b7ae00a9e3b139ae83ec8b21 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:21:26 -0700 Subject: [PATCH 103/477] usage example location --- src/libutil/error.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..d42781311 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -38,7 +38,7 @@ namespace nix { ErrorInfo structs are sent to the logger as part of an exception, or directly with the logError or logWarning macros. - See the error-demo.cc program for usage examples. + See libutil/tests/logging.cc for usage examples. */ From 8abb80a478116b10bf37162c71f602262de412a9 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Thu, 22 Oct 2020 23:59:01 -0500 Subject: [PATCH 104/477] Print built derivations as json for build Add --json option to nix build to allow machine readable output on stdout with all built derivations Fixes #1930 --- src/nix/build.cc | 6 +++++- src/nix/installables.cc | 27 +++++++++++++++++++++++++++ src/nix/installables.hh | 5 +++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 65708e98b..67be4024b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,9 +5,11 @@ #include "store-api.hh" #include "local-fs-store.hh" +#include + using namespace nix; -struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile +struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; BuildMode buildMode = bmNormal; @@ -86,6 +88,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile }, buildables[i]); updateProfile(buildables); + + if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 7473c9758..f385289e5 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -16,8 +16,35 @@ #include #include +#include + namespace nix { +nlohmann::json BuildableOpaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuildableFromDrv::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = path ? store->printStorePath(*path) : ""; + } + return res; +} + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const Buildable & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable); + } + return res; +} + void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, diff --git a/src/nix/installables.hh b/src/nix/installables.hh index c7c2f8981..f37b3f829 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -7,6 +7,8 @@ #include +#include + namespace nix { struct DrvInfo; @@ -16,11 +18,13 @@ namespace eval_cache { class EvalCache; class AttrCursor; } struct BuildableOpaque { StorePath path; + nlohmann::json toJSON(ref store) const; }; struct BuildableFromDrv { StorePath drvPath; std::map> outputs; + nlohmann::json toJSON(ref store) const; }; typedef std::variant< @@ -29,6 +33,7 @@ typedef std::variant< > Buildable; typedef std::vector Buildables; +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); struct App { From d52b12c0a53f88b8ea9238d604f293b54c8ae51a Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Fri, 23 Oct 2020 08:20:38 -0500 Subject: [PATCH 105/477] Test nix build --json --- tests/build.sh | 12 ++++++++++++ tests/local.mk | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/build.sh diff --git a/tests/build.sh b/tests/build.sh new file mode 100644 index 000000000..aa54b88eb --- /dev/null +++ b/tests/build.sh @@ -0,0 +1,12 @@ +source common.sh + +expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' +nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs.first | match(".*multiple-outputs-a-first")) and + (.outputs.second | match(".*multiple-outputs-a-second"))) + and (.[1] | + (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs.out | match(".*multiple-outputs-b"))) +' diff --git a/tests/local.mk b/tests/local.mk index a1929f96d..ce94ec80e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -35,7 +35,8 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ - content-addressed.sh + content-addressed.sh \ + build.sh # parallel.sh # build-remote-content-addressed-fixed.sh \ From 3edfe6090e9e15b205c21b19530607cbdcbbbe7a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:29:32 -0700 Subject: [PATCH 106/477] missing argument error --- src/libexpr/eval.cc | 12 ++++++++++-- src/libexpr/nixexpr.hh | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2b697882b..cf9f6c543 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -660,6 +660,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * }); } +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +{ + throw MissingArgumentError({ + .hint = hintfmt(s, s1), + .errPos = pos + }); +} + LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) { e.addTrace(std::nullopt, s, s2); @@ -1365,7 +1373,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default @@ -1373,7 +1381,7 @@ values, then all is well. But if the function arguments don't have default values, evaluation fails. -The classic case for this error is evaluating a nix file with nix-build that expects +The classic case for this error is evaluating a nix file that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..bf2cd1f15 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); +MakeError(MissingArgumentError, Error); MakeError(RestrictedPathError, Error); From 8895fa70a4b05ddebbb5a23ea96464d5e01345fb Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:05:21 -0700 Subject: [PATCH 107/477] pare down the error message --- src/libexpr/eval.cc | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cf9f6c543..540bfcf7b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1373,31 +1373,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -An 'auto-call' is when a nix expression is evaluated without any external arguments. -If that nix expression is a function, and that function's arguments all have default -values, then all is well. +Nix attempted to evaluate a function as a top level expression; in this case it must have all its +arguments supplied either by default values, or passed explicitly with --arg or --argstr. -But if the function arguments don't have default values, evaluation fails. +https://nixos.org/manual/nix/stable/#ss-functions)", i.name); -The classic case for this error is evaluating a nix file that expects -to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... - - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... - -More about callPackage: -https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From b327de9c2d5182e5814b2e956631b8794b45999b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:09:59 -0700 Subject: [PATCH 108/477] change message --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 540bfcf7b..3667ee6ba 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1375,7 +1375,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -Nix attempted to evaluate a function as a top level expression; in this case it must have all its +nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with --arg or --argstr. https://nixos.org/manual/nix/stable/#ss-functions)", i.name); From c4c3c15c19bc448a4797e5d9577539cc14890618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20H=C3=B6ppner?= Date: Thu, 12 Nov 2020 15:46:08 +0000 Subject: [PATCH 109/477] Fix default nix-path The default nix-path values for nixpkgs and root channels were incorrect. --- src/libexpr/eval.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5e3fcf4ac..c6f4d1716 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2104,10 +2104,19 @@ EvalSettings::EvalSettings() Strings EvalSettings::getDefaultNixPath() { Strings res; - auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathExists(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + add(getHome() + "/.nix-defexpr/channels"); - add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); - add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); + add(settings.nixStateDir + "/profiles/per-user/root/channels"); return res; } From ac5081d28035ef8d6a907fc3bff812d534719f74 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Nov 2020 17:49:27 +0100 Subject: [PATCH 110/477] nix-build: Fix #4197 output order regression --- src/nix-build/nix-build.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f60e0706c..74fafd426 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -487,6 +487,7 @@ static void main_nix_build(int argc, char * * argv) else { std::vector pathsToBuild; + std::vector> pathsToBuildOrdered; std::map> drvMap; @@ -498,6 +499,7 @@ static void main_nix_build(int argc, char * * argv) throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); pathsToBuild.push_back({drvPath, {outputName}}); + pathsToBuildOrdered.push_back({drvPath, {outputName}}); auto i = drvMap.find(drvPath); if (i != drvMap.end()) @@ -513,25 +515,23 @@ static void main_nix_build(int argc, char * * argv) std::vector outPaths; - for (auto & [drvPath, info] : drvMap) { - auto & [counter, wantedOutputs] = info; + for (auto & [drvPath, outputName] : pathsToBuildOrdered) { + auto & [counter, _wantedOutputs] = drvMap.at({drvPath}); std::string drvPrefix = outLink; if (counter) drvPrefix += fmt("-%d", counter + 1); auto builtOutputs = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : wantedOutputs) { - auto outputPath = builtOutputs.at(outputName); + auto outputPath = builtOutputs.at(outputName); - if (auto store2 = store.dynamic_pointer_cast()) { - std::string symlink = drvPrefix; - if (outputName != "out") symlink += "-" + outputName; - store2->addPermRoot(outputPath, absPath(symlink)); - } - - outPaths.push_back(outputPath); + if (auto store2 = store.dynamic_pointer_cast()) { + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; + store2->addPermRoot(outputPath, absPath(symlink)); } + + outPaths.push_back(outputPath); } logger->stop(); From d264da8d968bb3afe29def1f4aff0f91f25e36c6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Nov 2020 17:49:55 +0100 Subject: [PATCH 111/477] tests: Test #4197 nix-build output order regression --- tests/nix-build-examples.nix | 33 +++++++++++++++++++++++++++++++++ tests/nix-build.sh | 15 +++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tests/nix-build-examples.nix diff --git a/tests/nix-build-examples.nix b/tests/nix-build-examples.nix new file mode 100644 index 000000000..e54dbbf62 --- /dev/null +++ b/tests/nix-build-examples.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +rec { + + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + + input1 = mkDerivation { + name = "dependencies-input-1"; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + ''; + }; + + body = mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + meta.description = "Random test package"; + }; + +} diff --git a/tests/nix-build.sh b/tests/nix-build.sh index 3123c6da3..44a5a14cd 100644 --- a/tests/nix-build.sh +++ b/tests/nix-build.sh @@ -26,3 +26,18 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out) [[ $outPath = $outPath2 ]] + +# The order of the paths on stdout must correspond to the -A options +# https://github.com/NixOS/nix/issues/4197 + +input0="$(nix-build nix-build-examples.nix -A input0 --no-out-link)" +input1="$(nix-build nix-build-examples.nix -A input1 --no-out-link)" +input2="$(nix-build nix-build-examples.nix -A input2 --no-out-link)" +body="$(nix-build nix-build-examples.nix -A body --no-out-link)" + +outPathsA="$(echo $(nix-build nix-build-examples.nix -A input0 -A input1 -A input2 -A body --no-out-link))" +[[ "$outPathsA" = "$input0 $input1 $input2 $body" ]] + +# test a different ordering to make sure it fails, not just in 23 out of 24 permutations +outPathsB="$(echo $(nix-build nix-build-examples.nix -A body -A input1 -A input2 -A input0 --no-out-link))" +[[ "$outPathsB" = "$body $input1 $input2 $input0" ]] From 01db455733899728947a15f8d50942f10432d61e Mon Sep 17 00:00:00 2001 From: Jake Waksbaum Date: Mon, 16 Nov 2020 02:35:50 -0500 Subject: [PATCH 112/477] Fix deadlock in nix-store when max-connections=1 This fixes a bug I encountered where `nix-store -qR` will deadlock when the `--include-outputs` flag is passed and `max-connections=1`. The deadlock occurs because `RemoteStore::queryDerivationOutputs` takes the only connection from the connection pool and uses it to check the daemon version. If the version is new enough, it calls `Store::queryDerivationOutputs`, which eventually calls `RemoteStore::queryPartialDerivationOutputMap`, where we take another connection from the connection pool to check the version again. Because we still haven't released the connection from the caller, this waits for a connection to be available, causing a deadlock. This diff solves the issue by using `getProtocol` to check the protocol version in the caller `RemoteStore::queryDerivationOutputs`, which immediately frees the connection back to the pool before returning the protocol version. That way we've already freed the connection by the time we call `RemoteStore::queryPartialDerivationOutputMap`. --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..a392f3b8c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -409,10 +409,10 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { return Store::queryDerivationOutputs(path); } + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); From ef84c780bb901011e090b9f12d293d136193a428 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Nov 2020 16:26:29 +0100 Subject: [PATCH 113/477] filterANSIEscapes(): Handle UTF-8 characters --- src/libutil/tests/tests.cc | 10 ++++++++++ src/libutil/util.cc | 23 ++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index ffba832d8..35a5d27bb 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -587,4 +587,14 @@ namespace nix { ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); } + + TEST(filterANSIEscapes, utf8) { + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); + } + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 53342b5cb..01ab9111f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1409,7 +1409,28 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in i++; else { - t += *i++; w++; + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; } } From 7de21f6664ffd4a66e5ef70058a4c450985a1981 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 13 Nov 2020 17:00:32 +0100 Subject: [PATCH 114/477] Make the sql debug statements more useful Print the expanded sql query (with the variables bound to their value) rather than the original one in case of error --- src/libstore/sqlite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 31a1f0cac..f5935ee5c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -147,14 +147,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } From bccff827dc968b08bddda03aadcb3d9cc41c2719 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:50:36 +0100 Subject: [PATCH 115/477] Fix deadlock in IFD through the daemon Fixes #4235. --- src/libstore/derivations.cc | 6 ++++-- tests/remote-store.sh | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 517ecfaa2..eea129df3 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -463,13 +463,15 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath { auto h = drvHashes.find(drvPath); if (h == drvHashes.end()) { - assert(store.isValidPath(drvPath)); // Cache it h = drvHashes.insert_or_assign( drvPath, hashDerivationModulo( store, - store.readDerivation(drvPath), + parseDerivation( + store, + readFile(store.toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)), false)).first; } return h->second; diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 3a61946f9..f7ae1a2ed 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -7,6 +7,20 @@ nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor startDaemon +# Test import-from-derivation through the daemon. +[[ $(nix eval --impure --raw --expr ' + with import ./config.nix; + import ( + mkDerivation { + name = "foo"; + bla = import ./dependencies.nix; + buildCommand = " + echo \\\"hi\\\" > $out + "; + } + ) +') = hi ]] + storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh nix-store --dump-db > $TEST_ROOT/d1 From e6b7c7b79c697f1d8508930964e8c810f04a8963 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:58:55 +0100 Subject: [PATCH 116/477] Cleanup --- src/libstore/derivations.cc | 5 +---- src/libstore/local-store.cc | 10 ++-------- src/libstore/store-api.cc | 8 ++++++++ src/libstore/store-api.hh | 3 +++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index eea129df3..106ec5daa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -468,10 +468,7 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath drvPath, hashDerivationModulo( store, - parseDerivation( - store, - readFile(store.toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)), + store.readInvalidDerivation(drvPath), false)).first; } return h->second; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4e404dca..93d073768 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -624,10 +624,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = parseDerivation( - *this, - readFile(Store::toRealPath(info.path)), - Derivation::nameFromPath(info.path)); + auto drv = readInvalidDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1003,10 +1000,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, - parseDerivation( - *this, - readFile(Store::toRealPath(i.path)), - Derivation::nameFromPath(i.path))); + readInvalidDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..7f808f45a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1007,6 +1007,14 @@ Derivation Store::readDerivation(const StorePath & drvPath) } } +Derivation Store::readInvalidDerivation(const StorePath & drvPath) +{ + return parseDerivation( + *this, + readFile(Store::toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)); +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..30c9e7d0a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -611,6 +611,9 @@ public: /* Read a derivation (which must already be valid). */ Derivation readDerivation(const StorePath & drvPath); + /* Read a derivation from a potentially invalid path. */ + Derivation readInvalidDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If From 3daa256728e29f9059a79b8ecaf9325e52f1704a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 15:26:39 +0100 Subject: [PATCH 117/477] Remove tests.binaryTarball This test no longer works on Hydra because import-from-derivation is no longer allowed. --- flake.nix | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/flake.nix b/flake.nix index 8ff048be3..653cacded 100644 --- a/flake.nix +++ b/flake.nix @@ -388,38 +388,6 @@ inherit (self) overlay; }); - # Test whether the binary tarball works in an Ubuntu system. - tests.binaryTarball = - with nixpkgsFor.x86_64-linux; - vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu1204x86_64; - } - '' - set -x - useradd -m alice - su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*' - mkdir /dest-nix - mount -o bind /dest-nix /nix # Provide a writable /nix. - chown alice /nix - su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' - su - alice -c 'nix-store --verify' - su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}' - - # Check whether 'nix upgrade-nix' works. - cat > /tmp/paths.nix < Date: Tue, 17 Nov 2020 15:36:20 +0100 Subject: [PATCH 118/477] Remove stray debug statement This was causing a failure on macOS. https://hydra.nixos.org/build/130354318 --- tests/binary-cache.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 6415c9302..e3b3982fe 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -258,9 +258,6 @@ docPath=$(nix-store -q --references $outPath) # +---...-multi-output-doc nix copy --to "file://$cacheDir" $outPath -( echo $outPath $docPath - find $cacheDir -) >/tmp/blurb hashpart() { basename "$1" | cut -c1-32 From 0fa6d380b2143af68d4c7c6ab55b639516014304 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 18 Nov 2020 11:20:50 +0700 Subject: [PATCH 119/477] fix typo in comment in fetchurl.nix --- corepkgs/fetchurl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index a84777f57..02531103b 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -1,6 +1,6 @@ { system ? "" # obsolete , url -, hash ? "" # an SRI ash +, hash ? "" # an SRI hash # Legacy hash specification , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" From 2113ae2d856a208350ccbafdc19e8dda322515b8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Nov 2020 16:50:06 +0000 Subject: [PATCH 120/477] Make drv hash modulo memo table thread-safe Let's get one step closer to the daemon not needing to fork. --- src/libexpr/primops.cc | 7 ++++--- src/libstore/derivations.cc | 27 +++++++++++++++------------ src/libstore/derivations.hh | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 236433ef1..41f06c219 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1132,9 +1132,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * However, we don't bother doing this for floating CA derivations because their "hash modulo" is indeterminate until built. */ - if (drv.type() != DerivationType::CAFloating) - drvHashes.insert_or_assign(drvPath, - hashDerivationModulo(*state.store, Derivation(drv), false)); + if (drv.type() != DerivationType::CAFloating) { + auto h = hashDerivationModulo(*state.store, Derivation(drv), false); + drvHashes.lock()->insert_or_assign(drvPath, h); + } state.mkAttrs(v, 1 + drv.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 106ec5daa..231ca26c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -451,7 +451,7 @@ DerivationType BasicDerivation::type() const } -DrvHashes drvHashes; +Sync drvHashes; /* pathDerivationModulo and hashDerivationModulo are mutually recursive */ @@ -459,19 +459,22 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - auto h = drvHashes.find(drvPath); - if (h == drvHashes.end()) { - // Cache it - h = drvHashes.insert_or_assign( - drvPath, - hashDerivationModulo( - store, - store.readInvalidDerivation(drvPath), - false)).first; + { + auto hashes = drvHashes.lock(); + auto h = hashes->find(drvPath); + if (h != hashes->end()) { + return h->second; + } } - return h->second; + auto h = hashDerivationModulo( + store, + store.readInvalidDerivation(drvPath), + false); + // Cache it + drvHashes.lock()->insert_or_assign(drvPath, h); + return h; } /* See the header for interface details. These are the implementation details. diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f9ba935e6..b966d6d90 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -210,7 +210,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; -extern DrvHashes drvHashes; // FIXME: global, not thread-safe +// FIXME: global, though at least thread-safe. +extern Sync drvHashes; /* Memoisation of `readDerivation(..).resove()`. */ typedef std::map< From 0327580e5485143acb6eb7c8c515e3e179f1e3fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Nov 2020 20:30:41 +0100 Subject: [PATCH 121/477] Fix assertion failure in LockFile::LockFile() Fixes #4241. --- src/libexpr/flake/lockfile.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 8e2f7131f..6089d1363 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -78,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) { if (jsonNode.find("inputs") == jsonNode.end()) return; for (auto & i : jsonNode["inputs"].items()) { - if (i.value().is_array()) { + if (i.value().is_array()) { // FIXME: remove, obsolete InputPath path; for (auto & j : i.value()) path.push_back(j); @@ -87,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) std::string inputKey = i.value(); auto k = nodeMap.find(inputKey); if (k == nodeMap.end()) { - auto jsonNode2 = json["nodes"][inputKey]; - auto input = std::make_shared(jsonNode2); + auto nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = std::make_shared(*jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; - getInputs(*input, jsonNode2); + getInputs(*input, *jsonNode2); } if (auto child = std::dynamic_pointer_cast(k->second)) node.inputs.insert_or_assign(i.key(), child); From 4dcb183af31d5cb33b6ef8e581e77d1c892a58b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Nov 2020 20:59:36 +0100 Subject: [PATCH 122/477] AttrCursor::getStringWithContext(): Force re-evaluation if the cached context is not valid Fixes #4236. --- src/libexpr/eval-cache.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 381344b40..7b025be23 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -525,8 +525,17 @@ string_t AttrCursor::getStringWithContext() cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { - debug("using cached string attribute '%s'", getAttrPathStr()); - return *s; + bool valid = true; + for (auto & c : s->second) { + if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) { + valid = false; + break; + } + } + if (valid) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } } else throw TypeError("'%s' is not a string", getAttrPathStr()); } From c3bad73e27a5bda3f8f8c768cec818c52b0f95e3 Mon Sep 17 00:00:00 2001 From: Wil Taylor Date: Sat, 21 Nov 2020 14:28:49 +1000 Subject: [PATCH 123/477] Added switch --- src/nix/bundle.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2d0a0b6ea..c59018726 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -11,6 +11,7 @@ struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; std::optional outLink; + bool skipReferenceCheck = false; CmdBundle() { @@ -32,6 +33,12 @@ struct CmdBundle : InstallableCommand .handler = {&outLink}, .completer = completePath }); + + addFlag({ + .longName = "skip-refcheck", + .description = "Skip checking of references in bundle.", + .handler = {&skipReferenceCheck, true}, + }); } std::string description() override @@ -117,7 +124,7 @@ struct CmdBundle : InstallableCommand auto outPathS = store->printStorePath(outPath); auto info = store->queryPathInfo(outPath); - if (!info->references.empty()) + if (!info->references.empty() && !skipReferenceCheck) throw Error("'%s' has references; a bundler must not leave any references", outPathS); if (!outLink) From 233b61d3d6482544c35b9d340240bf3260acff13 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 6 Nov 2020 11:53:00 +0100 Subject: [PATCH 124/477] installer: simplify the per-build installation The goal is to allow the installation and testing of arbitrary Nix versions. Extend the base installer to accept a `--tarball-url-prefix ` to change where the Nix tarball is getting downloaded from. Once this is merged it should allow to: 1. Pick an evaluation at https://hydra.nixos.org/jobset/nix/master that looks healthy 2. Select the installedScript build and find the store path. Now equipped with all of this, use an instance of nar-serve to fetch the install script and release tarballs: curl -sfL https://nar-serve.numtide.com/nix/store/rkv4yh7pym941bhj0849zqdkg2546bdv-installer-script/install \ | sh --tarball-url-prefix https://nar-serve.numtide.com/nix/store Or with cachix, strip the /nix/store and derivation name and then: curl -sfL https://mycache.cachix.org/serve/rkv4yh7pym941bhj0849zqdkg2546bdv/install \ | sh --tarball-url-prefix https://mycache.cachix.org/serve Fixes #4047 --- flake.nix | 19 +++++++++++- scripts/install-nix-from-closure.sh | 3 ++ scripts/install.in | 46 ++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 653cacded..7311a2471 100644 --- a/flake.nix +++ b/flake.nix @@ -324,9 +324,26 @@ '' mkdir -p $out/nix-support + # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix + # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ") + (system: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + '' + ) [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${version} diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 2de41a6a4..6352a8fac 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -98,6 +98,9 @@ while [ $# -gt 0 ]; do echo "" echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" echo "" + if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then + echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from." + fi ) >&2 # darwin and Catalina+ diff --git a/scripts/install.in b/scripts/install.in index 9a281d776..7c3b795cc 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -25,18 +25,47 @@ require_util() { } case "$(uname -s).$(uname -m)" in - Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; - Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; - Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; - Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; - # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;; - Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + Linux.x86_64) + hash=@tarballHash_x86_64-linux@ + path=@tarballPath_x86_64-linux@ + system=x86_64-linux + ;; + Linux.i?86) + hash=@tarballHash_i686-linux@ + path=@tarballPath_i686-linux@ + system=i686-linux + ;; + Linux.aarch64) + hash=@tarballHash_aarch64-linux@ + path=@tarballPath_aarch64-linux@ + system=aarch64-linux + ;; + Darwin.x86_64) + hash=@tarballHash_x86_64-darwin@ + path=@tarballPath_x86_64-darwin@ + system=x86_64-darwin + ;; + Darwin.arm64) + hash=@binaryTarball_x86_64-darwin@ + path=@tarballPath_x86_64-darwin@ + # eventually maybe: arm64-darwin + system=x86_64-darwin + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" +# Use this command-line option to fetch the tarballs using nar-serve or Cachix +if "${1:---tarball-url-prefix}"; then + if [ -z "${2:-}" ]; then + oops "missing argument for --tarball-url-prefix" + fi + url=${2}/${path} + shift 2 +else + url=https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz +fi -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" +tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" @@ -68,6 +97,7 @@ tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) [ -e "$script" ] || oops "installation script is missing from the binary tarball!" +export INVOKED_FROM_INSTALL_IN=1 "$script" "$@" } # End of wrapping From df83b6df68a7a7d71747c1d6cfb98f6b4ee732f0 Mon Sep 17 00:00:00 2001 From: Kai Wohlfahrt Date: Sat, 21 Nov 2020 22:06:15 +0000 Subject: [PATCH 125/477] Return signatures in Perl path info --- perl/lib/Nix/Store.xs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..9e3b7d389 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -110,10 +110,14 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); - AV * arr = newAV(); + AV * refs = newAV(); for (auto & i : info->references) - av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) arr))); + av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); + XPUSHs(sv_2mortal(newRV((SV *) refs))); + AV * sigs = newAV(); + for (auto & i : info->sigs) + av_push(sigs, newSVpv(i.c_str(), 0)); + XPUSHs(sv_2mortal(newRV((SV *) sigs))); } catch (Error & e) { croak("%s", e.what()); } From 07603890d2867907905ba411cee550390d868936 Mon Sep 17 00:00:00 2001 From: Wil Taylor Date: Mon, 23 Nov 2020 21:19:40 +1000 Subject: [PATCH 126/477] Removed reference check from bundler command --- src/nix/bundle.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index c59018726..eddd82f40 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -11,7 +11,6 @@ struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; std::optional outLink; - bool skipReferenceCheck = false; CmdBundle() { @@ -34,11 +33,6 @@ struct CmdBundle : InstallableCommand .completer = completePath }); - addFlag({ - .longName = "skip-refcheck", - .description = "Skip checking of references in bundle.", - .handler = {&skipReferenceCheck, true}, - }); } std::string description() override @@ -123,10 +117,6 @@ struct CmdBundle : InstallableCommand auto outPathS = store->printStorePath(outPath); - auto info = store->queryPathInfo(outPath); - if (!info->references.empty() && !skipReferenceCheck) - throw Error("'%s' has references; a bundler must not leave any references", outPathS); - if (!outLink) outLink = baseNameOf(app.program); From 226116f48272dbe25bb9d3b8fa138cf6b10ec8ef Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 23 Nov 2020 16:12:33 +0000 Subject: [PATCH 127/477] fetchMercurial: set HGPLAIN when invoking hg Without setting HGPLAIN, the user's environment leaks into hg invocations, which means that the output may not be in the expected format. HGPLAIN is the Mercurial-recommended solution for this in that it's intended for uses by scripts and programs which are looking to parse Mercurial's output in a consistent manner. --- src/libfetchers/mercurial.cc | 54 ++++++++++++++++++++++++++++-------- tests/fetchMercurial.sh | 3 ++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 41f60c45c..07a51059d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -11,6 +11,36 @@ using namespace std::string_literals; namespace nix::fetchers { +namespace { + +RunOptions hgOptions(const Strings & args) { + RunOptions opts("hg", args); + opts.searchPath = true; + + auto env = getEnv(); + // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc. + env["HGPLAIN"] = ""; + opts.environment = env; + + return opts; +} + +// runProgram wrapper that uses hgOptions instead of stock RunOptions. +string runHg(const Strings & args, const std::optional & input = {}) +{ + RunOptions opts = hgOptions(args); + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + + return res.second; +} + +} + struct MercurialInputScheme : InputScheme { std::optional inputFromURL(const ParsedURL & url) override @@ -100,11 +130,11 @@ struct MercurialInputScheme : InputScheme assert(sourcePath); // FIXME: shut up if file is already tracked. - runProgram("hg", true, + runHg( { "add", *sourcePath + "/" + std::string(file) }); if (commitMsg) - runProgram("hg", true, + runHg( { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } @@ -130,7 +160,7 @@ struct MercurialInputScheme : InputScheme if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { - bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; if (!clean) { @@ -143,10 +173,10 @@ struct MercurialInputScheme : InputScheme if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); + input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, actualUrl)); @@ -224,33 +254,33 @@ struct MercurialInputScheme : InputScheme if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) + hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); if (pathExists(cacheDir)) { try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + runHg({ "recover", "-R", cacheDir }); + runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } else { throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir }); } } auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); @@ -263,7 +293,7 @@ struct MercurialInputScheme : InputScheme Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); + runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index af8ef8d5b..d8a4e09d2 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -15,6 +15,9 @@ hg init $repo echo '[ui]' >> $repo/.hg/hgrc echo 'username = Foobar ' >> $repo/.hg/hgrc +# Set ui.tweakdefaults to ensure HGPLAIN is being set. +echo 'tweakdefaults = True' >> $repo/.hg/hgrc + echo utrecht > $repo/hello touch $repo/.hgignore hg add --cwd $repo hello .hgignore From 5b0790355fe10b5cdc2468928a7cb4703cd0861a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 23 Nov 2020 17:40:17 +0100 Subject: [PATCH 128/477] Fix macOS sandbox build Since c4c3c15c19bc448a4797e5d9577539cc14890618 (#4251) building Nix for macOS with sandboxing fails: ``` getting status of /nix/var/nix/profiles/per-user/root/channels/nixpkgs: Operation not permitted ``` This happens, because `EvalSettings::getDefaultNixPath` tries to access paths outside the sandbox. Since the state-dir is not required for doc generation, it is set to the dummy folder. This needs to be done for all nix invocations during doc generation, as `EvalSettings::getDefaultNixPath` is called unconditionally. --- doc/manual/local.mk | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 7d9a1a3e8..bb8b3b60a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -15,7 +15,14 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 dist-files += $(man-pages) -nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr +# Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. +dummy-env = env -i \ + HOME=/dummy \ + NIX_CONF_DIR=/dummy \ + NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ + NIX_STATE_DIR=/dummy + +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @@ -45,11 +52,11 @@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix - $(trace-gen) $(bindir)/nix __dump-args > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @@ -58,7 +65,7 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix - $(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp mv $@.tmp $@ # Generate the HTML manual. From 437189e446e16399d347e4430c4d115b4cf2ddf1 Mon Sep 17 00:00:00 2001 From: Lily Ballard Date: Tue, 24 Nov 2020 14:12:32 -0800 Subject: [PATCH 129/477] Escape filename given to nix-shell in shebang mode This prevents spaces or other metacharacters from causing nix-shell to execute the wrong path. Fixes #4229. --- src/nix-build/nix-build.cc | 4 ++-- tests/nix-shell.sh | 18 +++++++++++++++++- tests/shell.nix | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 74fafd426..38048da52 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } } diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 7b2be650a..4775bafb9 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -47,6 +47,14 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode again with metacharacters in the filename. +# First word of filename is chosen to not match any file in the test root. +sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + # Test nix-shell shebang mode for ruby # This uses a fake interpreter that returns the arguments passed # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. @@ -54,7 +62,15 @@ sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shel chmod a+rx $TEST_ROOT/shell.shebang.rb output=$($TEST_ROOT/shell.shebang.rb abc ruby) -[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] + +# Test nix-shell shebang mode for ruby again with metacharacters in the filename. +# Note: fake interpreter only space-separates args without adding escapes to its output. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/shell.nix b/tests/shell.nix index 6ce59b416..24ebcc04c 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -50,7 +50,7 @@ let pkgs = rec { # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" {} '' mkdir -p $out/bin - echo 'printf -- "$*"' > $out/bin/ruby + echo 'printf %s "$*"' > $out/bin/ruby chmod a+rx $out/bin/ruby ''; From 13c557fe823549db75c3dc24c99c46e1c4e1378e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 11:20:03 +0100 Subject: [PATCH 130/477] fix the hash rewriting for ca-derivations --- src/libstore/build/derivation-goal.cc | 14 ++++++++++++++ tests/content-addressed.nix | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf2bad62c..0e4504857 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3121,6 +3121,20 @@ void DerivationGoal::registerOutputs() newInfo0.references = refs.second; if (refs.first) newInfo0.references.insert(newInfo0.path); + if (scratchPath != newInfo0.path) { + // Also rewrite the output path + auto source = sinkToSource([&](Sink & nextSink) { + StringSink sink; + dumpPath(actualPath, sink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + } assert(newInfo0.ca); return newInfo0; diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 8ca96d4bf..985220f48 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -16,14 +16,16 @@ rec { }; rootCA = mkDerivation { name = "rootCA"; - outputs = [ "out" "dev" ]; + outputs = [ "out" "dev" "foo"]; buildCommand = '' echo "building a CA derivation" echo "The seed is ${toString seed}" mkdir -p $out echo ${rootLegacy}/hello > $out/dep - # test symlink at root + ln -s $out $out/self + # test symlinks at root ln -s $out $dev + ln -s $out $foo ''; __contentAddressed = true; outputHashMode = "recursive"; @@ -34,7 +36,8 @@ rec { buildCommand = '' echo "building a dependent derivation" mkdir -p $out - echo ${rootCA}/hello > $out/dep + cat ${rootCA}/self/dep + echo ${rootCA}/self/dep > $out/dep ''; __contentAddressed = true; outputHashMode = "recursive"; From 0287f8305790a87b128ce09d7d3fa0de7104673c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 12:34:43 +0100 Subject: [PATCH 131/477] Ask for confirmation before allowing flake Nix configuration settings --- src/libexpr/flake/flake.cc | 69 ++++++++++++++++++++----------------- src/libexpr/flake/flake.hh | 2 +- src/libmain/progress-bar.cc | 11 ++++++ src/libutil/logging.hh | 3 ++ 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bdcf63c21..453d219dc 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -233,28 +233,28 @@ static Flake getFlake( if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); - for (auto & option : *nixConfig->value->attrs) { - forceTrivialValue(state, *option.value, *option.pos); - if (option.value->type == tString) - flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)}); - else if (option.value->type == tInt) - flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)}); - else if (option.value->type == tBool) - flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)}); - else if (option.value->isList()) { + for (auto & setting : *nixConfig->value->attrs) { + forceTrivialValue(state, *setting.value, *setting.pos); + if (setting.value->type == tString) + flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); + else if (setting.value->type == tInt) + flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); + else if (setting.value->type == tBool) + flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); + else if (setting.value->isList()) { std::vector ss; - for (unsigned int n = 0; n < option.value->listSize(); ++n) { - auto elem = option.value->listElems()[n]; + for (unsigned int n = 0; n < setting.value->listSize(); ++n) { + auto elem = setting.value->listElems()[n]; if (elem->type != tString) - throw TypeError("list element in flake configuration option '%s' is %s while a string is expected", - option.name, showType(*option.value)); - ss.push_back(state.forceStringNoCtx(*elem, *option.pos)); + throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", + setting.name, showType(*setting.value)); + ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); } - flake.config.options.insert({option.name, ss}); + flake.config.settings.insert({setting.name, ss}); } else - throw TypeError("flake configuration option '%s' is %s", - option.name, showType(*option.value)); + throw TypeError("flake configuration setting '%s' is %s", + setting.name, showType(*setting.value)); } } @@ -637,27 +637,34 @@ Flake::~Flake() { } void ConfigFile::apply() { - for (auto & [name, value] : options) { - // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU. - if (name != "bash-prompt-suffix" && - name != "bash-prompt" && - name != "substituters" && - name != "extra-substituters") - { - warn("ignoring untrusted flake configuration option '%s'", name); - continue; - } + std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; + + for (auto & [name, value] : settings) { + + auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; + // FIXME: Move into libutil/config.cc. + std::string valueS; if (auto s = std::get_if(&value)) - globalConfig.set(name, *s); + valueS = *s; else if (auto n = std::get_if(&value)) - globalConfig.set(name, fmt("%d", n)); + valueS = fmt("%d", n); else if (auto b = std::get_if>(&value)) - globalConfig.set(name, b->t ? "true" : "false"); + valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil + valueS = concatStringsSep(" ", *ss); // FIXME: evil else assert(false); + + if (!whitelist.count(baseName)) { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') { + warn("ignoring untrusted flake configuration setting '%s'", name); + continue; + } + } + + globalConfig.set(name, valueS); } } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 7eebd9044..65ed1ad0a 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -51,7 +51,7 @@ struct ConfigFile { using ConfigValue = std::variant, std::vector>; - std::map options; + std::map settings; void apply(); }; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 07b45b3b5..0e5432fca 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -466,6 +466,17 @@ public: Logger::writeToStdout(s); } } + + std::optional ask(std::string_view msg) override + { + auto state(state_.lock()); + if (!state->active || !isatty(STDIN_FILENO)) return {}; + std::cerr << fmt("\r\e[K%s ", msg); + auto s = trim(readLine(STDIN_FILENO)); + if (s.size() != 1) return {}; + draw(*state); + return s[0]; + } }; Logger * makeProgressBar(bool printBuildLogs) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index e3fe613e8..cd0cb64c5 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -106,6 +106,9 @@ public: formatHelper(f, args...); writeToStdout(f.str()); } + + virtual std::optional ask(std::string_view s) + { return {}; } }; ActivityId getCurActivity(); From 9a586e34ac3ae37bfd18f4e82af26df938ab9d96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 13:11:07 +0100 Subject: [PATCH 132/477] Record trusted/untrusted settings in ~/.local/share/nix --- src/libexpr/flake/flake.cc | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 453d219dc..0eadd94db 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,6 +6,8 @@ #include "fetchers.hh" #include "finally.hh" +#include + namespace nix { using namespace flake; @@ -635,6 +637,27 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } +// setting name -> setting value -> allow or ignore. +typedef std::map> TrustedList; + +Path trustedListPath() +{ + return getDataDir() + "/nix/trusted-settings.json"; +} + +static TrustedList readTrustedList() +{ + auto path = trustedListPath(); + if (!pathExists(path)) return {}; + auto json = nlohmann::json::parse(readFile(path)); + return json; +} + +static void writeTrustedList(const TrustedList & trustedList) +{ + writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); +} + void ConfigFile::apply() { std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; @@ -657,8 +680,28 @@ void ConfigFile::apply() assert(false); if (!whitelist.count(baseName)) { - // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') { + auto trustedList = readTrustedList(); + + bool trusted = false; + + if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + trusted = *saved; + } else { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { + if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = false; + writeTrustedList(trustedList); + } + } else { + if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = trusted = true; + writeTrustedList(trustedList); + } + } + } + + if (!trusted) { warn("ignoring untrusted flake configuration setting '%s'", name); continue; } From 8252a44e96619b6df0901a1f62e60b3d5951fd12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 13:16:36 +0100 Subject: [PATCH 133/477] Move to separate file --- src/libexpr/flake/config.cc | 81 +++++++++++++++++++++++++++++++++++++ src/libexpr/flake/flake.cc | 76 ---------------------------------- 2 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 src/libexpr/flake/config.cc diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc new file mode 100644 index 000000000..63566131e --- /dev/null +++ b/src/libexpr/flake/config.cc @@ -0,0 +1,81 @@ +#include "flake.hh" + +#include + +namespace nix::flake { + +// setting name -> setting value -> allow or ignore. +typedef std::map> TrustedList; + +Path trustedListPath() +{ + return getDataDir() + "/nix/trusted-settings.json"; +} + +static TrustedList readTrustedList() +{ + auto path = trustedListPath(); + if (!pathExists(path)) return {}; + auto json = nlohmann::json::parse(readFile(path)); + return json; +} + +static void writeTrustedList(const TrustedList & trustedList) +{ + writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); +} + +void ConfigFile::apply() +{ + std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; + + for (auto & [name, value] : settings) { + + auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; + + // FIXME: Move into libutil/config.cc. + std::string valueS; + if (auto s = std::get_if(&value)) + valueS = *s; + else if (auto n = std::get_if(&value)) + valueS = fmt("%d", n); + else if (auto b = std::get_if>(&value)) + valueS = b->t ? "true" : "false"; + else if (auto ss = std::get_if>(&value)) + valueS = concatStringsSep(" ", *ss); // FIXME: evil + else + assert(false); + + if (!whitelist.count(baseName)) { + auto trustedList = readTrustedList(); + + bool trusted = false; + + if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + trusted = *saved; + } else { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { + if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = false; + writeTrustedList(trustedList); + } + } else { + if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = trusted = true; + writeTrustedList(trustedList); + } + } + } + + if (!trusted) { + warn("ignoring untrusted flake configuration setting '%s'", name); + continue; + } + } + + globalConfig.set(name, valueS); + } +} + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0eadd94db..3e866e1f9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,8 +6,6 @@ #include "fetchers.hh" #include "finally.hh" -#include - namespace nix { using namespace flake; @@ -637,78 +635,4 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } -// setting name -> setting value -> allow or ignore. -typedef std::map> TrustedList; - -Path trustedListPath() -{ - return getDataDir() + "/nix/trusted-settings.json"; -} - -static TrustedList readTrustedList() -{ - auto path = trustedListPath(); - if (!pathExists(path)) return {}; - auto json = nlohmann::json::parse(readFile(path)); - return json; -} - -static void writeTrustedList(const TrustedList & trustedList) -{ - writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); -} - -void ConfigFile::apply() -{ - std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; - - for (auto & [name, value] : settings) { - - auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; - - // FIXME: Move into libutil/config.cc. - std::string valueS; - if (auto s = std::get_if(&value)) - valueS = *s; - else if (auto n = std::get_if(&value)) - valueS = fmt("%d", n); - else if (auto b = std::get_if>(&value)) - valueS = b->t ? "true" : "false"; - else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil - else - assert(false); - - if (!whitelist.count(baseName)) { - auto trustedList = readTrustedList(); - - bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { - trusted = *saved; - } else { - // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } - } - } - - if (!trusted) { - warn("ignoring untrusted flake configuration setting '%s'", name); - continue; - } - } - - globalConfig.set(name, valueS); - } -} - } From 1fd13d67e85b8365baed1cfb435870e24a7e6979 Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Thu, 26 Nov 2020 14:26:57 +0100 Subject: [PATCH 134/477] archive: disable preallocate-contents by default using fallocate() to preallocate files space does more harm than good: - breaks compression on btrfs - has been called "not the right thing to do" by xfs developers (because delayed allocation that most filesystems implement leads to smarter allocation than what the filesystem needs to do if we upfront fallocate files) --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f1479329f..03534abc4 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,7 +27,7 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, true, "preallocate-contents", + Setting preallocateContents{this, false, "preallocate-contents", "Whether to preallocate files when writing objects with known size."}; }; From 05d9442f68ba906ae50c12deab63fc1b9836b149 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Nov 2020 21:45:28 +0100 Subject: [PATCH 135/477] builtins.fetchGit: Fix shortRev attribute for dirty trees --- src/libexpr/primops/fetchTree.cc | 2 +- tests/fetchGit.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6f637a43..d094edf92 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -39,7 +39,7 @@ void emitTreeAttrs( // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev auto emptyHash = Hash(htSHA1); mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); } if (input.getType() == "git") diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index cedd796f7..76390fa59 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -59,6 +59,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath [[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]] # Fetching with a explicit hash should succeed. path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -132,6 +133,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).shortRev") = 0000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a From 9bd8184f1fb2e91ac4fb7207abe56f2a30a81d97 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 18:20:35 +0100 Subject: [PATCH 136/477] Allow fixed-output derivations to depend on (floating) content-addressed ones Fix an overlook of https://github.com/NixOS/nix/pull/4056 --- src/libstore/build/derivation-goal.cc | 4 ++-- tests/content-addressed.nix | 11 +++++++++++ tests/content-addressed.sh | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 0e4504857..76c49f92c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && - fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 985220f48..61079176f 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -63,4 +63,15 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentFixedOutput = mkDerivation { + name = "dependent-fixed-output"; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA="; + buildCommand = '' + cat ${dependentCA}/dep + echo foo > $out + ''; + + }; } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index bdab09c86..52f7529b5 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -40,6 +40,7 @@ testCutoff () { #testDerivation dependentCA testCutoffFor transitivelyDependentCA testCutoffFor dependentNonCA + testCutoffFor dependentFixedOutput } testGC () { From e224c16d28be738b260d9d15d0503da3febf99de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 14:14:11 +0100 Subject: [PATCH 137/477] Macro hygiene --- src/libutil/logging.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index cd0cb64c5..ff17056f4 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -178,8 +178,8 @@ extern Verbosity verbosity; /* suppress msgs > this */ lightweight status messages. */ #define logErrorInfo(level, errorInfo...) \ do { \ - if (level <= nix::verbosity) { \ - logger->logEI(level, errorInfo); \ + if ((level) <= nix::verbosity) { \ + logger->logEI((level), errorInfo); \ } \ } while (0) From 59276244738ebec32ed0593ee4af615852ff564d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Nov 2020 14:14:57 +0100 Subject: [PATCH 138/477] Lower verbosity for 'Failed to find a machine' message --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 9a0e9f08d..8348d8c91 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -201,7 +201,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep(", ", m.mandatoryFeatures); } - logErrorInfo(lvlInfo, { + logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, { .name = "Remote build", .description = "Failed to find a machine for remote build!", .hint = hint From 3b7e00ce2215b742d9fdb1b8d4a4d76d349028c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Nov 2020 14:36:15 +0100 Subject: [PATCH 139/477] Move primeCache() to Worker::run() We need the missing path info to communicate the worker's remaining goals to the progress bar. --- src/libstore/build/derivation-goal.hh | 12 +----------- src/libstore/build/local-store-build.cc | 22 ++-------------------- src/libstore/build/substitution-goal.hh | 8 +------- src/libstore/build/worker.cc | 21 ++++++++++++++++++++- 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 4976207e0..8ee0be9e1 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -40,9 +40,8 @@ struct InitialOutput { std::optional known; }; -class DerivationGoal : public Goal +struct DerivationGoal : public Goal { -private: /* Whether to use an on-disk .drv file. */ bool useDerivation; @@ -246,7 +245,6 @@ private: friend struct RestrictedStore; -public: DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -264,17 +262,11 @@ public: void work() override; - StorePath getDrvPath() - { - return drvPath; - } - /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); BuildResult getResult() { return result; } -private: /* The states. */ void getDerivation(); void loadDerivation(); @@ -318,8 +310,6 @@ private: /* Run the builder's process. */ void runChild(); - friend int childEntry(void *); - /* Check that the derivation outputs all exist and register them as valid. */ void registerOutputs(); diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc index a05fb5805..c91cda2fd 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/local-store-build.cc @@ -5,25 +5,10 @@ namespace nix { -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); - primeCache(*this, drvPaths); - Goals goals; for (auto & path : drvPaths) { if (path.path.isDerivation()) @@ -44,9 +29,8 @@ void LocalStore::buildPaths(const std::vector & drvPaths, ex = i->ex; } if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); + if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -84,8 +68,6 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {{path}}); - Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); Goals goals = {goal}; diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3ae9a9e6b..dee2cecbf 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,11 +8,8 @@ namespace nix { class Worker; -class SubstitutionGoal : public Goal +struct SubstitutionGoal : public Goal { - friend class Worker; - -private: /* The store path that should be realised through a substitute. */ StorePath storePath; @@ -56,7 +53,6 @@ private: /* Content address for recomputing store path */ std::optional ca; -public: SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); @@ -82,8 +78,6 @@ public: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 17c10cd71..1f8999a4b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -207,7 +207,26 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - for (auto & i : _topGoals) topGoals.insert(i); + std::vector topPaths; + + for (auto & i : _topGoals) { + topGoals.insert(i); + if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + } else if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->storePath}); + } + } + + /* Call queryMissing() efficiently query substitutes. */ + StorePathSet willBuild, willSubstitute, unknown; + uint64_t downloadSize, narSize; + store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); debug("entered goal loop"); From c0d1354b7da9ffc2923bc102abb67d03b655fbbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Nov 2020 12:17:07 +0100 Subject: [PATCH 140/477] Macro hygiene --- src/libutil/logging.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index ff17056f4..82ba54051 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -191,8 +191,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ arguments are evaluated lazily. */ #define printMsg(level, args...) \ do { \ - if (level <= nix::verbosity) { \ - logger->log(level, fmt(args)); \ + auto __lvl = level; \ + if (__lvl <= nix::verbosity) { \ + logger->log(__lvl, fmt(args)); \ } \ } while (0) From 88798613ee288c7a801dcc1e73723a10a385df38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Nov 2020 14:59:03 +0100 Subject: [PATCH 141/477] replaceStrings(): Use std::string_view --- src/libutil/util.cc | 6 +++--- src/libutil/util.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 01ab9111f..c1b12e725 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1273,11 +1273,11 @@ string trim(const string & s, const string & whitespace) } -string replaceStrings(const std::string & s, +string replaceStrings(std::string_view s, const std::string & from, const std::string & to) { - if (from.empty()) return s; - string res = s; + string res(s); + if (from.empty()) return res; size_t pos = 0; while ((pos = res.find(from, pos)) != std::string::npos) { res.replace(pos, from.size(), to); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index cafe93702..117fe86e7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -383,7 +383,7 @@ string trim(const string & s, const string & whitespace = " \n\r\t"); /* Replace all occurrences of a string inside another string. */ -string replaceStrings(const std::string & s, +string replaceStrings(std::string_view s, const std::string & from, const std::string & to); From 438977731cf049cf47873d5825456d47a1aac541 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 1 Dec 2020 14:57:56 +0100 Subject: [PATCH 142/477] shut up clang warnings - Fix some class/struct discrepancies - Explicit the overloading of `run` in the `Cmd*` classes - Ignore a warning in the generated lexer --- src/libexpr/lexer.l | 2 ++ src/libstore/build/goal.hh | 2 +- src/libstore/build/worker.hh | 4 ++-- src/libstore/local-store.hh | 4 ++-- src/nix/run.cc | 8 ++++++++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index f6e83926b..225ab3287 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -12,6 +12,8 @@ %{ +#pragma clang diagnostic ignored "-Wunneeded-internal-declaration" + #include #include "nixexpr.hh" diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0781a9d38..fca4f2d00 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -7,7 +7,7 @@ namespace nix { /* Forward definition. */ struct Goal; -struct Worker; +class Worker; /* A pointer to a goal. */ typedef std::shared_ptr GoalPtr; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 3a53a8def..bf8cc4586 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -8,8 +8,8 @@ namespace nix { /* Forward definition. */ -class DerivationGoal; -class SubstitutionGoal; +struct DerivationGoal; +struct SubstitutionGoal; /* Workaround for not being able to declare a something like diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d4435220d..58ec93f27 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -296,8 +296,8 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend class DerivationGoal; - friend class SubstitutionGoal; + friend struct DerivationGoal; + friend struct SubstitutionGoal; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index 790784382..92a52c6cd 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -22,6 +22,9 @@ std::string chrootHelperName = "__run_in_chroot"; struct RunCommon : virtual Command { + + using Command::run; + void runProgram(ref store, const std::string & program, const Strings & args) @@ -59,6 +62,9 @@ struct RunCommon : virtual Command struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { + + using InstallablesCommand::run; + std::vector command = { getEnv("SHELL").value_or("bash") }; CmdShell() @@ -144,6 +150,8 @@ static auto rCmdShell = registerCommand("shell"); struct CmdRun : InstallableCommand, RunCommon { + using InstallableCommand::run; + std::vector args; CmdRun() From aa684861127eabc18a0d7386a66c5b75d4962897 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 12:43:52 +0100 Subject: [PATCH 143/477] writeFull/writeFile: Use std::string_view --- src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c1b12e725..98a069c3e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -320,7 +320,7 @@ void readFile(const Path & path, Sink & sink) } -void writeFile(const Path & path, const string & s, mode_t mode) +void writeFile(const Path & path, std::string_view s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -663,7 +663,7 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr } -void writeFull(int fd, const string & s, bool allowInterrupts) +void writeFull(int fd, std::string_view s, bool allowInterrupts) { writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 117fe86e7..eb872dd4b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -106,7 +106,7 @@ string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s, mode_t mode = 0666); +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); void writeFile(const Path & path, Source & source, mode_t mode = 0666); @@ -157,7 +157,7 @@ void replaceSymlink(const Path & target, const Path & link, requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); -void writeFull(int fd, const string & s, bool allowInterrupts = true); +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); From faa31f40846f7a4dbc2487d000b112a6aef69d1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:00:43 +0100 Subject: [PATCH 144/477] Sink: Use std::string_view --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/build/derivation-goal.cc | 8 +- src/libstore/daemon.cc | 4 +- src/libstore/filetransfer.cc | 18 ++-- src/libstore/filetransfer.hh | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/references.cc | 50 +++++------- src/libstore/references.hh | 4 +- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/store-api.cc | 4 +- src/libutil/archive.cc | 18 ++-- src/libutil/archive.hh | 8 +- src/libutil/compression.cc | 113 +++++++++++++------------- src/libutil/hash.cc | 18 ++-- src/libutil/hash.hh | 2 +- src/libutil/serialise.cc | 48 +++++------ src/libutil/serialise.hh | 52 +++++------- src/libutil/util.cc | 30 +++---- src/libutil/util.hh | 1 - src/nix/make-content-addressable.cc | 4 +- 21 files changed, 182 insertions(+), 211 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f6224d6a0..a918b7208 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) promise.set_exception(std::current_exception()); } }}); - auto data = promise.get_future().get(); - sink((unsigned char *) data->data(), data->size()); + sink(*promise.get_future().get()); } std::shared_ptr BinaryCacheStore::getFile(const std::string & path) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 76c49f92c..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -916,10 +916,8 @@ void DerivationGoal::buildDone() LogSink(Activity & act) : act(act) { } - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - + void operator() (std::string_view data) override { + for (auto c : data) { if (c == '\n') { flushLine(); } else { @@ -3127,7 +3125,7 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 60cca4fda..cb214bee3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -153,10 +153,10 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) + void operator () (std::string_view data) { to << STDERR_WRITE; - writeString(data, len, to); + writeString(data, to); } }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c2c65af05..31b4215a9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) - , finalSink([this](const unsigned char * data, size_t len) { + , finalSink([this](std::string_view data) { if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); /* Only write data to the sink if this is a successful response. */ if (successfulStatuses.count(httpStatus)) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + writtenToSink += data.size(); + this->request.dataCallback(data); } } else - this->result.data->append((char *) data, len); + this->result.data->append(data); }) { if (!request.expectedETag.empty()) @@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer } if (errorSink) - (*errorSink)((unsigned char *) contents, realSize); - (*decompressionSink)((unsigned char *) contents, realSize); + (*errorSink)({(char *) contents, realSize}); + (*decompressionSink)({(char *) contents, realSize}); return realSize; } catch (...) { @@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) state->request.notify_one(); }); - request.dataCallback = [_state](char * buf, size_t len) { + request.dataCallback = [_state](std::string_view data) { auto state(_state->lock()); @@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) /* Append data to the buffer and wake up the calling thread. */ - state->data.append(buf, len); + state->data.append(data); state->avail.notify_one(); }; @@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) if it's blocked on a full buffer. We don't hold the state lock while doing this to prevent blocking the download thread if sink() takes a long time. */ - sink((unsigned char *) chunk.data(), chunk.size()); + sink(chunk); } } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index c89c51a21..afc7e7aa6 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -61,7 +61,7 @@ struct FileTransferRequest bool decompress = true; std::shared_ptr data; std::string mimeType; - std::function dataCallback; + std::function dataCallback; FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index a9efdd0b6..e35031f7a 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor parents.top()->start = pos; } - void receiveContents(unsigned char * data, size_t len) override + void receiveContents(std::string_view data) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libstore/references.cc b/src/libstore/references.cc index d2096cb49..eb117b5ba 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -55,27 +55,23 @@ struct RefScanSink : Sink RefScanSink() { } - void operator () (const unsigned char * data, size_t len); + void operator () (std::string_view data) override + { + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + std::string(data, 0, refLength); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search((const unsigned char *) data.data(), data.size(), hashes, seen); + + size_t tailLen = data.size() <= refLength ? data.size() : refLength; + tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)); + tail.append({data.data() + data.size() - tailLen, tailLen}); + } }; -void RefScanSink::operator () (const unsigned char * data, size_t len) -{ - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + string((const char *) data, len > refLength ? refLength : len); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search(data, len, hashes, seen); - - size_t tailLen = len <= refLength ? len : refLength; - tail = - string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + - string((const char *) data + len - tailLen, tailLen); -} - - std::pair scanForReferences(const string & path, const PathSet & refs) { @@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S assert(from.size() == to.size()); } -void RewritingSink::operator () (const unsigned char * data, size_t len) +void RewritingSink::operator () (std::string_view data) { std::string s(prev); - s.append((const char *) data, len); + s.append(data); size_t j = 0; while ((j = s.find(from, j)) != string::npos) { @@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len) pos += consumed; - if (consumed) nextSink((unsigned char *) s.data(), consumed); + if (consumed) nextSink(s.substr(0, consumed)); } void RewritingSink::flush() { if (prev.empty()) return; pos += prev.size(); - nextSink((unsigned char *) prev.data(), prev.size()); + nextSink(prev); prev.clear(); } @@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) { } -void HashModuloSink::operator () (const unsigned char * data, size_t len) +void HashModuloSink::operator () (std::string_view data) { - rewritingSink(data, len); + rewritingSink(data); } HashResult HashModuloSink::finish() @@ -176,10 +172,8 @@ HashResult HashModuloSink::finish() NAR with self-references and a NAR with some of the self-references already zeroed out do not produce a hash collision. FIXME: proof. */ - for (auto & pos : rewritingSink.matches) { - auto s = fmt("|%d", pos); - hashSink((unsigned char *) s.data(), s.size()); - } + for (auto & pos : rewritingSink.matches) + hashSink(fmt("|%d", pos)); auto h = hashSink.finish(); return {h.first, rewritingSink.pos}; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c2efd095c..4f12e6b21 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -19,7 +19,7 @@ struct RewritingSink : Sink RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; void flush(); }; @@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink HashModuloSink(HashType ht, const std::string & modulus); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; HashResult finish() override; }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48af9f0c4..6e0c85237 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -857,7 +857,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (!source) throw Error("no source"); size_t len = readNum(from); auto buf = std::make_unique(len); - writeString(buf.get(), source->read(buf.get(), len), to); + writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 552c4aac7..4519dd5b5 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -398,7 +398,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, res.data->size(), res.durationMs); - sink((unsigned char *) res.data->data(), res.data->size()); + sink(*res.data); } else throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3b2be05cb..27be66cac 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -772,8 +772,8 @@ void copyStorePath(ref srcStore, ref dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink progressSink([&](const unsigned char * data, size_t len) { - total += len; + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); act.progress(total, info->narSize); }); TeeSink tee { sink, progressSink }; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 03534abc4..046ef841b 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -57,7 +57,7 @@ static void dumpContents(const Path & path, size_t size, auto n = std::min(left, buf.size()); readFull(fd.get(), buf.data(), n); left -= n; - sink(buf.data(), n); + sink({(char *) buf.data(), n}); } writePadding(size, sink); @@ -162,7 +162,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents(buf.data(), n); + sink.receiveContents({(char *) buf.data(), n}); left -= n; } @@ -300,21 +300,21 @@ struct RestoreSink : ParseSink Path dstPath; AutoCloseFD fd; - void createDirectory(const Path & path) + void createDirectory(const Path & path) override { Path p = dstPath + path; if (mkdir(p.c_str(), 0777) == -1) throw SysError("creating directory '%1%'", p); }; - void createRegularFile(const Path & path) + void createRegularFile(const Path & path) override { Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); if (!fd) throw SysError("creating file '%1%'", p); } - void isExecutable() + void isExecutable() override { struct stat st; if (fstat(fd.get(), &st) == -1) @@ -323,7 +323,7 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(uint64_t len) + void preallocateContents(uint64_t len) override { if (!archiveSettings.preallocateContents) return; @@ -341,12 +341,12 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, size_t len) + void receiveContents(std::string_view data) override { - writeFull(fd.get(), data, len); + writeFull(fd.get(), data); } - void createSymlink(const Path & path, const string & target) + void createSymlink(const Path & path, const string & target) override { Path p = dstPath + path; nix::createSymlink(target, p); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 5665732d2..fe22435ef 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -58,7 +58,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(unsigned char * data, size_t len) { }; + virtual void receiveContents(std::string_view data) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; @@ -72,14 +72,14 @@ struct RetrieveRegularNARSink : ParseSink RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - void createDirectory(const Path & path) + void createDirectory(const Path & path) override { regular = false; } - void receiveContents(unsigned char * data, size_t len) + void receiveContents(std::string_view data) override { - sink(data, len); + sink(data); } void createSymlink(const Path & path, const string & target) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a117ddc72..986ba2976 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -22,18 +22,17 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; - while (len) { - size_t n = std::min(CHUNK_SIZE, len); - writeInternal(data, n); - data += n; - len -= n; + while (!data.empty()) { + size_t n = std::min(CHUNK_SIZE, data.size()); + writeInternal(data); + data.remove_prefix(n); } } - virtual void writeInternal(const unsigned char * data, size_t len) = 0; + virtual void writeInternal(std::string_view data) = 0; }; struct NoneSink : CompressionSink @@ -41,7 +40,7 @@ struct NoneSink : CompressionSink Sink & nextSink; NoneSink(Sink & nextSink) : nextSink(nextSink) { } void finish() override { flush(); } - void write(const unsigned char * data, size_t len) override { nextSink(data, len); } + void write(std::string_view data) override { nextSink(data); } }; struct GzipDecompressionSink : CompressionSink @@ -75,28 +74,28 @@ struct GzipDecompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (Bytef *) data; - strm.avail_in = len; + strm.next_in = (Bytef *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); int ret = inflate(&strm,Z_SYNC_FLUSH); if (ret != Z_OK && ret != Z_STREAM_END) throw CompressionError("error while decompressing gzip file: %d (%d, %d)", - zError(ret), len, strm.avail_in); + zError(ret), data.size(), strm.avail_in); finished = ret == Z_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (Bytef *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -130,25 +129,25 @@ struct XzDecompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - strm.next_in = data; - strm.avail_in = len; + strm.next_in = (const unsigned char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); if (ret != LZMA_OK && ret != LZMA_STREAM_END) throw CompressionError("error %d while decompressing xz file", ret); finished = ret == LZMA_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -181,15 +180,15 @@ struct BzipDecompressionSink : ChunkedCompressionSink void finish() override { flush(); - write(nullptr, 0); + write({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char *) data.data(); + strm.avail_in = data.size(); while (strm.avail_in) { checkInterrupt(); @@ -201,7 +200,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink finished = ret == BZ_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -230,17 +229,17 @@ struct BrotliDecompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - const uint8_t * next_in = data; - size_t avail_in = len; + auto next_in = (const uint8_t *) data.data(); + size_t avail_in = data.size(); uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); - while (!finished && (!data || avail_in)) { + while (!finished && (!data.data() || avail_in)) { checkInterrupt(); if (!BrotliDecoderDecompressStream(state, @@ -250,7 +249,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink throw CompressionError("error while decompressing brotli file"); if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); + nextSink({(char *) outbuf, sizeof(outbuf) - avail_out}); next_out = outbuf; avail_out = sizeof(outbuf); } @@ -338,25 +337,25 @@ struct XzCompressionSink : CompressionSink void finish() override { CompressionSink::flush(); - write(nullptr, 0); + write({}); } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { - strm.next_in = data; - strm.avail_in = len; + strm.next_in = (const unsigned char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); if (ret != LZMA_OK && ret != LZMA_STREAM_END) throw CompressionError("error %d while compressing xz file", ret); finished = ret == LZMA_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -389,27 +388,27 @@ struct BzipCompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - assert(len <= std::numeric_limits::max()); + assert(data.size() <= std::numeric_limits::max()); - strm.next_in = (char *) data; - strm.avail_in = len; + strm.next_in = (char *) data.data(); + strm.avail_in = data.size(); - while (!finished && (!data || strm.avail_in)) { + while (!finished && (!data.data() || strm.avail_in)) { checkInterrupt(); - int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH); if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) throw CompressionError("error %d while compressing bzip2 file", ret); finished = ret == BZ_STREAM_END; if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } @@ -439,28 +438,28 @@ struct BrotliCompressionSink : ChunkedCompressionSink void finish() override { flush(); - writeInternal(nullptr, 0); + writeInternal({}); } - void writeInternal(const unsigned char * data, size_t len) override + void writeInternal(std::string_view data) override { - const uint8_t * next_in = data; - size_t avail_in = len; + auto next_in = (const uint8_t *) data.data(); + size_t avail_in = data.size(); uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); - while (!finished && (!data || avail_in)) { + while (!finished && (!data.data() || avail_in)) { checkInterrupt(); if (!BrotliEncoderCompressStream(state, - data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in, &avail_out, &next_out, nullptr)) throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink(outbuf, sizeof(outbuf) - avail_out); + nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out}); next_out = outbuf; avail_out = sizeof(outbuf); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 8efff190a..4df8b4ecb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -291,12 +291,12 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, size_t len) + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); + if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } @@ -314,7 +314,7 @@ Hash hashString(HashType ht, std::string_view s) Ctx ctx; Hash hash(ht); start(ht, ctx); - update(ht, ctx, (const unsigned char *) s.data(), s.length()); + update(ht, ctx, s); finish(ht, ctx, hash.hash); return hash; } @@ -341,10 +341,10 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(const unsigned char * data, size_t len) +void HashSink::write(std::string_view data) { - bytes += len; - update(ht, *ctx, data, len); + bytes += data.size(); + update(ht, *ctx, data); } HashResult HashSink::finish() diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 6d6eb70ca..1b626dd85 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -156,7 +156,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len) override; + void write(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 038ede049..534ad5cbb 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -11,23 +11,23 @@ namespace nix { -void BufferedSink::operator () (const unsigned char * data, size_t len) +void BufferedSink::operator () (std::string_view data) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); - while (len) { + while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + len >= bufSize) { + if (bufPos + data.size() >= bufSize) { flush(); - write(data, len); + write(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; - memcpy(buffer.get() + bufPos, data, n); - data += n; bufPos += n; len -= n; + size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + memcpy(buffer.get() + bufPos, data.data(), n); + data.remove_prefix(n); bufPos += n; if (bufPos == bufSize) flush(); } } @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write(buffer.get(), n); + write({buffer.get(), n}); } @@ -59,9 +59,9 @@ static void warnLargeDump() } -void FdSink::write(const unsigned char * data, size_t len) +void FdSink::write(std::string_view data) { - written += len; + written += data.size(); static bool warned = false; if (warn && !warned) { if (written > threshold) { @@ -70,7 +70,7 @@ void FdSink::write(const unsigned char * data, size_t len) } } try { - writeFull(fd, data, len); + writeFull(fd, data); } catch (SysError & e) { _good = false; throw; @@ -101,7 +101,7 @@ void Source::drainInto(Sink & sink) size_t n; try { n = read(buf.data(), buf.size()); - sink(buf.data(), n); + sink({(char *) buf.data(), n}); } catch (EndOfFile &) { break; } @@ -229,9 +229,9 @@ std::unique_ptr sinkToSource( { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) yield(std::string(data)); + }); fun(sink); }); @@ -260,22 +260,22 @@ void writePadding(size_t len, Sink & sink) if (len % 8) { unsigned char zero[8]; memset(zero, 0, sizeof(zero)); - sink(zero, 8 - (len % 8)); + sink({(char *) zero, 8 - (len % 8)}); } } -void writeString(const unsigned char * buf, size_t len, Sink & sink) +void writeString(std::string_view data, Sink & sink) { - sink << len; - sink(buf, len); - writePadding(len, sink); + sink << data.size(); + sink(data); + writePadding(data.size(), sink); } Sink & operator << (Sink & sink, const string & s) { - writeString((const unsigned char *) s.data(), s.size(), sink); + writeString(s, sink); return sink; } @@ -394,14 +394,14 @@ Error readError(Source & source) } -void StringSink::operator () (const unsigned char * data, size_t len) +void StringSink::operator () (std::string_view data) { static bool warned = false; if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s->append((const char *) data, len); + s->append(data); } size_t ChainSource::read(unsigned char * data, size_t len) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5c7d3ce76..f9ab79997 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -14,19 +14,14 @@ namespace nix { struct Sink { virtual ~Sink() { } - virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual void operator () (std::string_view data) = 0; virtual bool good() { return true; } - - void operator () (const std::string & s) - { - (*this)((const unsigned char *) s.data(), s.size()); - } }; /* Just throws away data. */ struct NullSink : Sink { - void operator () (const unsigned char * data, size_t len) override + void operator () (std::string_view data) override { } }; @@ -35,21 +30,16 @@ struct NullSink : Sink struct BufferedSink : virtual Sink { size_t bufSize, bufPos; - std::unique_ptr buffer; + std::unique_ptr buffer; BufferedSink(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPos(0), buffer(nullptr) { } - void operator () (const unsigned char * data, size_t len) override; - - void operator () (const std::string & s) - { - Sink::operator()(s); - } + void operator () (std::string_view data) override; void flush(); - virtual void write(const unsigned char * data, size_t len) = 0; + virtual void write(std::string_view data) = 0; }; @@ -119,7 +109,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(const unsigned char * data, size_t len) override; + void write(std::string_view data) override; bool good() override; @@ -163,7 +153,7 @@ struct StringSink : Sink s->reserve(reservedSize); }; StringSink(ref s) : s(s) { }; - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; }; @@ -182,10 +172,10 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (const unsigned char * data, size_t len) + virtual void operator () (std::string_view data) { - sink1(data, len); - sink2(data, len); + sink1(data); + sink2(data); } }; @@ -200,7 +190,7 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, n); + sink({(char *) data, n}); return n; } }; @@ -241,24 +231,24 @@ struct LengthSink : Sink { uint64_t length = 0; - virtual void operator () (const unsigned char * _, size_t len) + void operator () (std::string_view data) override { - length += len; + length += data.size(); } }; /* Convert a function into a sink. */ struct LambdaSink : Sink { - typedef std::function lambda_t; + typedef std::function lambda_t; lambda_t lambda; LambdaSink(const lambda_t & lambda) : lambda(lambda) { } - virtual void operator () (const unsigned char * data, size_t len) + void operator () (std::string_view data) override { - lambda(data, len); + lambda(data); } }; @@ -302,7 +292,7 @@ std::unique_ptr sinkToSource( void writePadding(size_t len, Sink & sink); -void writeString(const unsigned char * buf, size_t len, Sink & sink); +void writeString(std::string_view s, Sink & sink); inline Sink & operator << (Sink & sink, uint64_t n) { @@ -315,7 +305,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) buf[5] = (n >> 40) & 0xff; buf[6] = (n >> 48) & 0xff; buf[7] = (unsigned char) (n >> 56) & 0xff; - sink(buf, sizeof(buf)); + sink({(char *) buf, sizeof(buf)}); return sink; } @@ -484,7 +474,7 @@ struct FramedSink : nix::BufferedSink } } - void write(const unsigned char * data, size_t len) override + void write(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -493,8 +483,8 @@ struct FramedSink : nix::BufferedSink ex = nullptr; std::rethrow_exception(ex2); } - to << len; - to(data, len); + to << data.size(); + to(data); }; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 98a069c3e..3d4fc4c25 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -346,7 +346,7 @@ void writeFile(const Path & path, Source & source, mode_t mode) while (true) { try { auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), (unsigned char *) buf.data(), n); + writeFull(fd.get(), {(char *) buf.data(), n}); } catch (EndOfFile &) { break; } } } catch (Error & e) { @@ -648,24 +648,16 @@ void readFull(int fd, unsigned char * buf, size_t count) } -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) -{ - while (count) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, (char *) buf, count); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) { - count -= res; - buf += res; - } - } -} - - void writeFull(int fd, std::string_view s, bool allowInterrupts) { - writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } } @@ -705,7 +697,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("reading from file"); } else if (rd == 0) break; - else sink(buf.data(), rd); + else sink({(char *) buf.data(), (size_t) rd}); } } @@ -1153,7 +1145,7 @@ void runProgram2(const RunOptions & options) } catch (EndOfFile &) { break; } - writeFull(in.writeSide.get(), buf.data(), n); + writeFull(in.writeSide.get(), {(char *) buf.data(), n}); } promise.set_value(); } catch (...) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index eb872dd4b..fcb7cad39 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -156,7 +156,6 @@ void replaceSymlink(const Path & target, const Path & link, /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); -void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index df3ec5194..12c2cf776 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -73,7 +73,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON *sink.s = rewriteStrings(*sink.s, rewrites); HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + hashModuloSink(*sink.s); auto narHash = hashModuloSink.finish().first; @@ -94,7 +94,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); From 1b79b5b983a6c775766bd0d1c7881042188998b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:10:56 +0100 Subject: [PATCH 145/477] read(): Use char * instead of unsigned char * This gets rid of some pointless casts. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libutil/archive.cc | 8 +++--- src/libutil/serialise.cc | 34 +++++++++++------------ src/libutil/serialise.hh | 44 +++++++++++++++--------------- src/libutil/tarfile.cc | 2 +- src/libutil/util.cc | 12 ++++---- src/libutil/util.hh | 2 +- 11 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cb214bee3..e5cfe94cb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -165,7 +165,7 @@ struct TunnelSource : BufferedSource Source & from; BufferedSink & to; TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } - size_t readUnbuffered(unsigned char * data, size_t len) override + size_t readUnbuffered(char * data, size_t len) override { to << STDERR_READ << len; to.flush(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 93d073768..348e5d0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1143,7 +1143,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, dump.resize(oldSize + want); auto got = 0; try { - got = source.read((uint8_t *) dump.data() + oldSize, want); + got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index e35031f7a..1427a0f98 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { auto n = source.read(data, len); pos += n; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 2d02a181b..63bde92de 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -75,7 +75,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) throw SysError("seeking in '%s'", cacheFile); std::string buf(length, 0); - readFull(fd.get(), (unsigned char *) buf.data(), length); + readFull(fd.get(), buf.data(), length); return buf; }); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e0c85237..be29f8e6f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -856,7 +856,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * else if (msg == STDERR_READ) { if (!source) throw Error("no source"); size_t len = readNum(from); - auto buf = std::make_unique(len); + auto buf = std::make_unique(len); writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 046ef841b..ed0eb2fb5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -50,14 +50,14 @@ static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(65536); + std::vector buf(65536); size_t left = size; while (left > 0) { auto n = std::min(left, buf.size()); readFull(fd.get(), buf.data(), n); left -= n; - sink({(char *) buf.data(), n}); + sink({buf.data(), n}); } writePadding(size, sink); @@ -155,14 +155,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size); uint64_t left = size; - std::vector buf(65536); + std::vector buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents({(char *) buf.data(), n}); + sink.receiveContents({buf.data(), n}); left -= n; } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 534ad5cbb..87c1099a1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -84,7 +84,7 @@ bool FdSink::good() } -void Source::operator () (unsigned char * data, size_t len) +void Source::operator () (char * data, size_t len) { while (len) { size_t n = read(data, len); @@ -96,12 +96,12 @@ void Source::operator () (unsigned char * data, size_t len) void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::vector buf(8192); while (true) { size_t n; try { n = read(buf.data(), buf.size()); - sink({(char *) buf.data(), n}); + sink({buf.data(), n}); } catch (EndOfFile &) { break; } @@ -117,9 +117,9 @@ std::string Source::drain() } -size_t BufferedSource::read(unsigned char * data, size_t len) +size_t BufferedSource::read(char * data, size_t len) { - if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + if (!buffer) buffer = decltype(buffer)(new char[bufSize]); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); @@ -138,12 +138,12 @@ bool BufferedSource::hasData() } -size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(char * data, size_t len) { ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, len); + n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } @@ -158,10 +158,10 @@ bool FdSource::good() } -size_t StringSource::read(unsigned char * data, size_t len) +size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); - size_t n = s.copy((char *) data, len, pos); + size_t n = s.copy(data, len, pos); pos += n; return n; } @@ -225,7 +225,7 @@ std::unique_ptr sinkToSource( std::string cur; size_t pos = 0; - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { if (!coro) coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { @@ -244,7 +244,7 @@ std::unique_ptr sinkToSource( } auto n = std::min(cur.size() - pos, len); - memcpy(data, (unsigned char *) cur.data() + pos, n); + memcpy(data, cur.data() + pos, n); pos += n; return n; @@ -258,9 +258,9 @@ std::unique_ptr sinkToSource( void writePadding(size_t len, Sink & sink) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; memset(zero, 0, sizeof(zero)); - sink({(char *) zero, 8 - (len % 8)}); + sink({zero, 8 - (len % 8)}); } } @@ -321,7 +321,7 @@ Sink & operator << (Sink & sink, const Error & ex) void readPadding(size_t len, Source & source) { if (len % 8) { - unsigned char zero[8]; + char zero[8]; size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) @@ -330,7 +330,7 @@ void readPadding(size_t len, Source & source) } -size_t readString(unsigned char * buf, size_t max, Source & source) +size_t readString(char * buf, size_t max, Source & source) { auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); @@ -345,7 +345,7 @@ string readString(Source & source, size_t max) auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); - source((unsigned char*) res.data(), len); + source(res.data(), len); readPadding(len, source); return res; } @@ -404,7 +404,7 @@ void StringSink::operator () (std::string_view data) s->append(data); } -size_t ChainSource::read(unsigned char * data, size_t len) +size_t ChainSource::read(char * data, size_t len) { if (useSecond) { return source2.read(data, len); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index f9ab79997..5bbbc7ce3 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -51,12 +51,12 @@ struct Source /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. It blocks until all the requested data is available, or throws an error if it is not going to be available. */ - void operator () (unsigned char * data, size_t len); + void operator () (char * data, size_t len); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -71,18 +71,18 @@ struct Source struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; - std::unique_ptr buffer; + std::unique_ptr buffer; BufferedSource(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; bool hasData(); protected: /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + virtual size_t readUnbuffered(char * data, size_t len) = 0; }; @@ -138,7 +138,7 @@ struct FdSource : BufferedSource bool good() override; protected: - size_t readUnbuffered(unsigned char * data, size_t len) override; + size_t readUnbuffered(char * data, size_t len) override; private: bool _good = true; }; @@ -163,7 +163,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; }; @@ -187,10 +187,10 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(unsigned char * data, size_t len) + size_t read(char * data, size_t len) { size_t n = orig.read(data, len); - sink({(char *) data, n}); + sink({data, n}); return n; } }; @@ -202,7 +202,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(unsigned char * data, size_t len) + size_t read(char * data, size_t len) { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -216,7 +216,7 @@ struct SizedSource : Source /* Consume the original source until no remain data is left to consume. */ size_t drainAll() { - std::vector buf(8192); + std::vector buf(8192); size_t sum = 0; while (this->remain > 0) { size_t n = read(buf.data(), buf.size()); @@ -256,13 +256,13 @@ struct LambdaSink : Sink /* Convert a function into a source. */ struct LambdaSource : Source { - typedef std::function lambda_t; + typedef std::function lambda_t; lambda_t lambda; LambdaSource(const lambda_t & lambda) : lambda(lambda) { } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { return lambda(data, len); } @@ -278,7 +278,7 @@ struct ChainSource : Source : source1(s1), source2(s2) { } - size_t read(unsigned char * data, size_t len) override; + size_t read(char * data, size_t len) override; }; @@ -322,7 +322,7 @@ template T readNum(Source & source) { unsigned char buf[8]; - source(buf, sizeof(buf)); + source((char *) buf, sizeof(buf)); uint64_t n = ((uint64_t) buf[0]) | @@ -354,7 +354,7 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); -size_t readString(unsigned char * buf, size_t max, Source & source); +size_t readString(char * buf, size_t max, Source & source); string readString(Source & source, size_t max = std::numeric_limits::max()); template T readStrings(Source & source); @@ -386,9 +386,9 @@ struct StreamToSourceAdapter : Source : istream(istream) { } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { - if (!istream->read((char *) data, len)) { + if (!istream->read(data, len)) { if (istream->eof()) { if (istream->gcount() == 0) throw EndOfFile("end of file"); @@ -411,7 +411,7 @@ struct FramedSource : Source { Source & from; bool eof = false; - std::vector pending; + std::vector pending; size_t pos = 0; FramedSource(Source & from) : from(from) @@ -423,13 +423,13 @@ struct FramedSource : Source while (true) { auto n = readInt(from); if (!n) break; - std::vector data(n); + std::vector data(n); from(data.data(), n); } } } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { if (eof) throw EndOfFile("reached end of FramedSource"); @@ -439,7 +439,7 @@ struct FramedSource : Source eof = true; return 0; } - pending = std::vector(len); + pending = std::vector(len); pos = 0; from(pending.data(), len); } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index c4d8a4f91..2da169ba7 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -66,7 +66,7 @@ private: *buffer = self->buffer.data(); try { - return self->source->read(self->buffer.data(), 4096); + return self->source->read((char *) self->buffer.data(), 4096); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3d4fc4c25..e6b6d287d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -340,13 +340,13 @@ void writeFile(const Path & path, Source & source, mode_t mode) if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(64 * 1024); + std::vector buf(64 * 1024); try { while (true) { try { auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {(char *) buf.data(), n}); + writeFull(fd.get(), {buf.data(), n}); } catch (EndOfFile &) { break; } } } catch (Error & e) { @@ -632,11 +632,11 @@ void replaceSymlink(const Path & target, const Path & link, } -void readFull(int fd, unsigned char * buf, size_t count) +void readFull(int fd, char * buf, size_t count) { while (count) { checkInterrupt(); - ssize_t res = read(fd, (char *) buf, count); + ssize_t res = read(fd, buf, count); if (res == -1) { if (errno == EINTR) continue; throw SysError("reading from file"); @@ -1137,7 +1137,7 @@ void runProgram2(const RunOptions & options) in.readSide = -1; writerThread = std::thread([&]() { try { - std::vector buf(8 * 1024); + std::vector buf(8 * 1024); while (true) { size_t n; try { @@ -1145,7 +1145,7 @@ void runProgram2(const RunOptions & options) } catch (EndOfFile &) { break; } - writeFull(in.writeSide.get(), {(char *) buf.data(), n}); + writeFull(in.writeSide.get(), {buf.data(), n}); } promise.set_value(); } catch (...) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fcb7cad39..0f82bed78 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -155,7 +155,7 @@ void replaceSymlink(const Path & target, const Path & link, /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ -void readFull(int fd, unsigned char * buf, size_t count); +void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); From a8a96dbaf8ae4f49491903876ddaba44be9d3a39 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Dec 2020 14:23:38 +0100 Subject: [PATCH 146/477] Add forgotten `override` annotation --- src/libutil/archive.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fe22435ef..9e9e11b1a 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -82,7 +82,7 @@ struct RetrieveRegularNARSink : ParseSink sink(data); } - void createSymlink(const Path & path, const string & target) + void createSymlink(const Path & path, const string & target) override { regular = false; } From 0d9e1af695ac238b0f8c8bc296bf01eb1430e26b Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Dec 2020 14:33:20 +0100 Subject: [PATCH 147/477] Remove an `unknown pragma` gcc warning --- src/libexpr/lexer.l | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 225ab3287..7298419d9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -12,7 +12,9 @@ %{ +#ifdef __clang__ #pragma clang diagnostic ignored "-Wunneeded-internal-declaration" +#endif #include From d8fc1bb7b00dd7b13d667d3cb41bfcbe0df699d0 Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 2 Dec 2020 10:15:18 -0500 Subject: [PATCH 148/477] fix tokens documentation --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index eabd83e3f..4655ca058 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -867,7 +867,7 @@ public: Example `~/.config/nix/nix.conf`: ``` - access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk ``` Example `~/code/flake.nix`: From 44da19f73cfa87bb09c14fe64fce97904db0970c Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Wed, 2 Dec 2020 17:00:32 +0100 Subject: [PATCH 149/477] Adds Nix CLI Guideline to docs As we are working towards Nix 3.0 we want to make sure that we make a huge step forward in Nix's user experience. And once 3.0 is out of the door we need to make sure that all future commands and features keep up the standard of user experience. This PR adds a CLI guideline document to the Nix documentation. Consider this document a good starting point and a checklist when somebody will be (re)implementing commands. Clearly this guideline does nothing to improve user experience on its own and can only be useful as long as it is going to be read and cared for. But it is a first step into that direction. --- doc/manual/src/SUMMARY.md | 1 + doc/manual/src/command-ref/cli-guideline.md | 571 ++++++++++++++++++++ 2 files changed, 572 insertions(+) create mode 100644 doc/manual/src/command-ref/cli-guideline.md diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md index 8281f683f..27a08bb43 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md @@ -63,6 +63,7 @@ - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) - [nix](command-ref/nix.md) + - [CLI guideline](command-ref/cli-guideline.md) - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) diff --git a/doc/manual/src/command-ref/cli-guideline.md b/doc/manual/src/command-ref/cli-guideline.md new file mode 100644 index 000000000..01df07136 --- /dev/null +++ b/doc/manual/src/command-ref/cli-guideline.md @@ -0,0 +1,571 @@ +# CLI guideline + +## Goals + +Purpose of this document is to provide a clear direction to **help design +delightful command line** experience. This document contain guidelines to +follow to ensure a consistent and approachable user experience. + +## Overview + +`nix` command provides a single entry to a number of sub-commands that help +**developers and system administrators** in the life-cycle of a software +project. We particularly need to pay special attention to help and assist new +users of Nix. + +# Naming the `COMMANDS` + +Words matter. Naming is an important part of the usability. Users will be +interacting with Nix on a regular basis so we should **name things for ease of +understanding**. + +We recommend following the [Principle of Least +Astonishment](https://en.wikipedia.org/wiki/Principle_of_least_astonishment). +This means that you should **never use acronyms or abbreviations** unless they +are commonly used in other tools (e.g. `nix init`). And if the command name is +too long (> 10-12 characters) then shortening it makes sense (e.g. +“prioritization” → “priority”). + +Commands should **follow a noun-verb dialogue**. Although noun-verb formatting +seems backwards from a speaking perspective (i.e. `nix store copy` vs. `nix +copy store`) it allows us to organize commands the same way users think about +completing an action (the group first, then the command). + +## Naming rules + +Rules are there to guide you by limiting your options. But not everything can +fit the rules all the time. In those cases document the exceptions in [Appendix +1: Commands naming exceptions](#appendix-1-commands-naming-exceptions) and +provide reason. The rules want to force a Nix developer to look, not just at +the command at hand, but also the command in a full context alongside other +`nix` commands. + +```shell +$ nix [] [] [] +``` + +- `GROUP`, `COMMAND`, `ARGUMENTS` and `OPTIONS` should be lowercase and in a + singular form. +- `GROUP` should be a **NOUN**. +- `COMMAND` should be a **VERB**. +- `ARGUMENTS` and `OPTIONS` are discussed in [*Input* section](#input). + +## Classification + +Some commands are more important, some less. While we want all of our commands +to be perfect we can only spend limited amount of time testing and improving +them. + +This classification tries to separate commands in 3 categories in terms of +their importance in regards to the new users. Users who are likely to be +impacted the most by bad user experience. This does not mean that we will not +take care of Tier3 commands, it means we will only put more helpful details for +beginners into Tier1 commands. + +- **Tier1**: Commands used for our main use cases and most likely used by new + users. From Tier1 commands we expect attention to details, such as: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) + and [aligning of text](#text-alignment). + - [Autocomplete](#shell-completion) of options. + - Show [next possible steps](#next-steps). + - Showing some [“tips”](#educate-the-user) when running logs running tasks + (eg. building / downloading) in order to teach users interesting bits of + Nix ecosystem. + - [Help pages](#help-is-essential) to be as good as we can write them + pointing to external documentation and tutorials for more. +- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really + exposing some implementation detail, but not something that we expect a user. + From Tier2 command we expect less attention to details, but still some: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) + and [aligning of text](#text-alignment). + - [Autocomplete](#shell-completion) of options. +- **Tier3**: Commands that expose certain internal functionality of `nix`, + mostly used by other scripts. + - [Autocomplete](#shell-completion) of options. + +# Help is essential + +Help should be built into your command line so that new users can gradually +discover new features when they need them. + +## Looking for help + +Since there is no standard way how user will look for help we rely on ways help +is provided by commonly used tools. As a guide for this we took `git` and +whenever in doubt look at it as a preferred direction. + +The rules are: + +- Help is shown by using `--help` or `help` command (eg `nix` `--``help` or + `nix help`). +- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show + a summary** of most common use cases. Summary is presented on the STDOUT + without any use of PAGER. +- For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the + man page of that command. By default the PAGER is used (as in `git`). +- At the end of either summary or man page there should be an URL pointing to + an online version of more detailed documentation. +- The structure of summaries and man pages should be the same as in `git`. + +## Anticipate where help is needed + +Even better then requiring the user to search for help is to anticipate and +predict when user might need it. Either because the lack of discoverability, +typo in the input or simply taking the opportunity to teach the user of +interesting - but less visible - details. + +### Shell completion + +This type of help is most common and almost expected by users. We need to +**provide the best shell completion** for `bash`, `zsh` and `fish`. + +Completion needs to be **context aware**, this mean when a user types: + +```shell +$ nix build n +``` + +we need to display a list of flakes starting with `n`. + +### Wrong input + +As we all know we humans make mistakes, all the time. When a typo - intentional +or unintentional - is made, we should prompt for closest possible options or +point to the documentation which would educate user to not make the same +errors. Here are few examples: + +In first example we prompt the user for typing wrong command name: + + +```shell +$ nix int +------------------------------------------------------------------------ + Error! Command `int` not found. +------------------------------------------------------------------------ + Did you mean: + |> nix init + |> nix input +``` + +Sometimes users will make mistake either because of a typo or simply because of +lack of discoverability. Our handling of this cases needs to be context +sensitive. + + +```shell +$ nix init --template=template#pyton +------------------------------------------------------------------------ + Error! Template `template#pyton` not found. +------------------------------------------------------------------------ +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> template#pyton + template#python-pip + template#python-poetry +``` + +### Next steps + +It can be invaluable to newcomers to show what a possible next steps and what +is the usual development workflow with Nix. For example: + + +```shell +$ nix init --template=template#python +Initializing project `template#python` + in `/home/USER/dev/new-project` + + Next steps + |> nix develop -- to enter development environment + |> nix build -- to build your project +``` + +### Educate the user + +We should take any opportunity to **educate users**, but at the same time we +must **be very very careful to not annoy users**. There is a thin line between +being helpful and being annoying. + +An example of educating users might be to provide *Tips* in places where they +are waiting. + +```shell +$ nix build + Started building my-project 1.2.3 + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +Now **Learn** part of the output is where you educate users. You should only +show it when you know that a build will take some time and not annoy users of +the builds that take only few seconds. + +Every feature like this should go though a intensive review and testing to +collect as much a feedback as possible and to fine tune every little detail. If +done right this can be an awesome features beginners and advance users will +love, but if not done perfectly it will annoy users and leave bad impression. + +# Input + +Input to a command is provided via `ARGUMENTS` and `OPTIONS`. + +`ARGUMENTS` represent a required input for a function. When choosing to use +`ARGUMENT` over function please be aware of the downsides that come with it: + +- User will need to remember the order of `ARGUMENTS`. This is not a problem if + there is only one `ARGUMENT`. +- With `OPTIONS` it is possible to provide much better auto completion. +- With `OPTIONS` it is possible to provide much better error message. +- Using `OPTIONS` it will mean there is a little bit more typing. + +We don’t discourage the use of `ARGUMENTS`, but simply want to make every +developer consider the downsides and choose wisely. + +## Naming the `OPTIONS` + +Then only naming convention - apart from the ones mentioned in Naming the +`COMMANDS` section is how flags are named. + +Flags are a type of `OPTION` that represent an option that can be turned ON of +OFF. We can say **flags are boolean type of** `**OPTION**`. + +Here are few examples of flag `OPTIONS`: + +- `--colors` vs. `--no-colors` (showing colors in the output) +- `--emojis` vs. `--no-emojis` (showing emojis in the output) + +## Prompt when input not provided + +For **Tier1** commands we want command to improve the discoverability of +possible input. A new user will most likely not know which `ARGUMENTS` and +`OPTIONS` are required or which values are possible for those options. + +In cases, the user might not provide the input or they provide wrong input, +rather then show the error, prompt a user with an option to find and select +correct input (see examples). + +Prompting is of course not required when TTY is not attached to STDIN. This +would mean that scripts wont need to handle prompt, but rather handle errors. + +A place to use prompt and provide user with interactive select + + +```shell +$ nix init +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> py + template#python-pip + template#python-poetry + [ Showing 2 templates from 1345 templates ] +``` + +Another great place to add prompts are **confirmation dialogues for dangerous +actions**. For example when adding new substitutor via `OPTIONS` or via +`flake.nix` we should prompt - for the first time - and let user review what is +going to happen. + + +```shell +$ nix build --option substitutors https://cache.example.org +------------------------------------------------------------------------ + Warning! A security related question need to be answered. +------------------------------------------------------------------------ + The following substitutors will be used to in `my-project`: + - https://cache.example.org + + Do you allow `my-project` to use above mentioned substitutors? + [y/N] |> y +``` + +# Output + +Terminal output can be quite limiting in many ways. Which should forces us to +think about the experience even more. As with every design the output is a +compromise between being terse and being verbose, between showing help to +beginners and annoying advance users. For this it is important that we know +what are the priorities. + +Nix command line should be first and foremost written with beginners in mind. +But users wont stay beginners for long and what was once useful might quickly +become annoying. There is no golden rule that we can give in this guideline +that would make it easier how to draw a line and find best compromise. + +What we would encourage is to **build prototypes**, do some **user testing** +and collect **feedback**. Then repeat the cycle few times. + +First design the *happy path* and only after your iron it out, continue to work +on **edge cases** (handling and displaying errors, changes of the output by +certain `OPTIONS`, etc…) + +## Follow best practices + +Needless to say we Nix must be a good citizen and follow best practices in +command line. + +In short: **STDOUT is for output, STDERR is for (human) messaging.** + +STDOUT and STDERR provide a way for you to output messages to the user while +also allowing them to redirect content to a file. For example: + +```shell +$ nix build > build.txt +------------------------------------------------------------------------ + Error! Atrribute `bin` missing at (1:94) from string. +------------------------------------------------------------------------ + + 1| with import { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } "" +``` + +Because this warning is on STDERR, it doesn’t end up in the file. + +But not everything on STDERR is an error though. For example, you can run `nix +build` and collect logs in a file while still seeing the progress. + +``` +$ nix build > build.txt + Evaluated 1234 files in 1.2 seconds + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +## Errors (WIP) + +**TODO**: Once we have implementation for the *happy path* then we will think +how to present errors. + +## Not only for humans + +Terse, machine-readable output formats can also be useful but shouldn’t get in +the way of making beautiful CLI output. When needed, commands should offer a +`--json` flag to allow users to easily parse and script the CLI. + +When TTY is not detected on STDOUT we should remove all design elements (no +colors, no emojis and using ASCII instead of Unicode symbols). The same should +happen when TTY is not detected on STDERR. We should not display progress / +status section, but only print warnings and errors. + +## Dialog with the user + +CLIs don't always make it clear when an action has taken place. For every +action a user performs, your CLI should provide an equal and appropriate +reaction, clearly highlighting the what just happened. For example: + +```shell +$ nix build + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +... + Success! You have successfully built my-project. +$ +``` + +Above command clearly states that command successfully completed. And in case +of `nix build`, which is a command that might take some time to complete, it is +equally important to also show that a command started. + +## Text alignment + +Text alignment is the number one design element that will present all of the +Nix commands as a family and not as separate tools glued together. + +The format we should follow is: + +```shell +$ nix COMMAND + VERB_1 NOUN and other words + VERB__1 NOUN and other words + |> Some details +``` + +Few rules that we can extract from above example: + +- Each line should start at least with one space. +- First word should be a VERB and must be aligned to the right. +- Second word should be a NOUN and must be aligned to the left. +- If you can not find a good VERB / NOUN pair, don’t worry make it as + understandable to the user as possible. +- More details of each line can be provided by `|>` character which is serving + as the first word when aligning the text + +Don’t forget you should also test your terminal output with colors and emojis +off (`--no-colors --no-emojis`). + +## Dim / Bright + +After comparing few terminals with different color schemes we would **recommend +to avoid using dimmed text**. The difference from the rest of the text is very +little in many terminal and color scheme combinations. Sometimes the difference +is not even notable, therefore relying on it wouldn’t make much sense. + +**The bright text is much better supported** across terminals and color +schemes. Most of the time the difference is perceived as if the bright text +would be bold. + +## Colors + +Humans are already conditioned by society to attach certain meaning to certain +colors. While the meaning is not universal, a simple collection of colors is +used to represent basic emotions. + +Colors that can be used in output + +- Red = error, danger, stop +- Green = success, good +- Yellow/Orange = proceed with caution, warning, in progress +- Blue/Magenta = stability, calm + +While colors are nice, when command line is used by machines (in automation +scripts) you want to remove the colors. There should be a global `--no-colors` +option that would remove the colors. + +## Special (Unicode) characters + +Most of the terminal have good support for Unicode characters and you should +use them in your output by default. But always have a backup solution that is +implemented only with ASCII characters and will be used when `--ascii` option +is going to be passed in. Please make sure that you test your output also +without Unicode characters + +More they showing all the different Unicode characters it is important to +**establish common set of characters** that we use for certain situations. + +## Emojis + +Emojis help channel emotions even better than text, colors and special +characters. + +We recommend **keeping the set of emojis to a minimum**. This will enable each +emoji to stand out more. + +As not everybody is happy about emojis we should provide an `--no-emojis` +option to disable them. Please make sure that you test your output also without +emojis. + +## Tables + +All commands that are listing certain data can be implemented in some sort of a +table. It’s important that each row of your output is a single ‘entry’ of data. +Never output table borders. It’s noisy and a huge pain for parsing using other +tools such as `grep`. + +Be mindful of the screen width. Only show a few columns by default with the +table header, for more the table can be manipulated by the following options: + +- `--no-headers`: Show column headers by default but allow to hide them. +- `--columns`: Comma-separated list of column names to add. +- `--sort`: Allow sorting by column. Allow inverse and multi-column sort as well. + +## Interactive output + +Interactive output was selected to be able to strike the balance between +beginners and advance users. While the default output will target beginners it +can, with a few key strokes, be changed into and advance introspection tool. + +### Progress + +For longer running commands we should provide and overview of the progress. +This is shown best in `nix build` example: + +```shell +$ nix build + Started building my-project 1.2.3 + Downloaded python3.8-poetry 1.2.3 in 5.3 seconds + Downloaded python3.8-requests 1.2.3 in 5.3 seconds +------------------------------------------------------------------------ + Press `v` to increase logs verbosity + |> `?` to see other options +------------------------------------------------------------------------ + Learn something new with every build... + |> See last logs of a build with `nix log --last` command. +------------------------------------------------------------------------ + Evaluated my-project 1.2.3 in 14.43 seconds +Downloading [12 / 200] + |> firefox 1.2.3 [#########> ] 10Mb/s | 2min left + Building [2 / 20] + |> glibc 1.2.3 -> buildPhase: +------------------------------------------------------------------------ +``` + +### Search + +Use a `fzf` like fuzzy search when there are multiple options to choose from. + +```shell +$ nix init +Initializing Nix project at `/path/to/here`. + Select a template for you new project: + |> py + template#python-pip + template#python-poetry + [ Showing 2 templates from 1345 templates ] +``` + +### Prompt + +In some situations we need to prompt the user and inform the user about what is +going to happen. + +```shell +$ nix build --option substitutors https://cache.example.org +------------------------------------------------------------------------ + Warning! A security related question need to be answered. +------------------------------------------------------------------------ + The following substitutors will be used to in `my-project`: + - https://cache.example.org + + Do you allow `my-project` to use above mentioned substitutors? + [y/N] |> y +``` + +## Verbosity + +There are many ways that you can control verbosity. + +Verbosity levels are: + +- `ERROR` (level 0) +- `WARN` (level 1) +- `NOTICE` (level 2) +- `INFO` (level 3) +- `TALKATIVE` (level 4) +- `CHATTY` (level 5) +- `DEBUG` (level 6) +- `VOMIT` (level 7) + +The default level that the command starts is `ERROR`. The simplest way to +increase the verbosity by stacking `-v` option (eg: `-vvv == level 3 == INFO`). +There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and +`--quiet` to run in `ERROR` verbosity level. + +---------- + +# Appendix 1: Commands naming exceptions + +`nix init` and `nix repl` are well established From 148608ba6ddf93168e86525627bed755a474d21f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 12:26:09 +0100 Subject: [PATCH 150/477] Add 'nix help' --- src/nix/main.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5056ceb78..a75f8ae65 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -149,6 +149,50 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs } }; +static void showHelp(std::vector subcommand) +{ + showManPage(subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand))); +} + +struct CmdHelp : Command +{ + std::vector subcommand; + + CmdHelp() + { + expectArgs({ + .label = "subcommand", + .handler = {&subcommand}, + }); + } + + std::string description() override + { + return "show help about 'nix' or a particular subcommand"; + } + + Examples examples() override + { + return { + Example{ + "To show help about 'nix' in general:", + "nix help" + }, + Example{ + "To show help about a particular subcommand:", + "nix help run" + }, + }; + } + + void run() override + { + showHelp(subcommand); + } +}; + +static auto rCmdHelp = registerCommand("help"); + void mainWrapped(int argc, char * * argv) { /* The chroot helper needs to be run before any threads have been From df552a26452f4cdf734edcac049187b8fd806153 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 21:25:32 +0100 Subject: [PATCH 151/477] nix eval: Add option to write a directory This is useful for generating the nix manpages, but it may have other applications (like generating configuration files without a Nix store). --- src/nix/eval.cc | 55 ++++++++++++++++++++++++++++++++++++++++++---- tests/pure-eval.sh | 8 +++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 8da81d667..0f02919de 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -3,6 +3,7 @@ #include "shared.hh" #include "store-api.hh" #include "eval.hh" +#include "eval-inline.hh" #include "json.hh" #include "value-to-json.hh" #include "progress-bar.hh" @@ -13,6 +14,7 @@ struct CmdEval : MixJSON, InstallableCommand { bool raw = false; std::optional apply; + std::optional writeTo; CmdEval() { @@ -24,6 +26,13 @@ struct CmdEval : MixJSON, InstallableCommand .labels = {"expr"}, .handler = {&apply}, }); + + addFlag({ + .longName = "write-to", + .description = "write a string or attrset of strings to 'path'", + .labels = {"path"}, + .handler = {&writeTo}, + }); } std::string description() override @@ -66,7 +75,7 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state).first; + auto [v, pos] = installable->toValue(*state); PathSet context; if (apply) { @@ -77,13 +86,51 @@ struct CmdEval : MixJSON, InstallableCommand v = vRes; } - if (raw) { + if (writeTo) { + stopProgressBar(); + + if (pathExists(*writeTo)) + throw Error("path '%s' already exists", *writeTo); + + std::function recurse; + + recurse = [&](Value & v, const Pos & pos, const Path & path) + { + state->forceValue(v); + if (v.type == tString) + // FIXME: disallow strings with contexts? + writeFile(path, v.string.s); + else if (v.type == tAttrs) { + if (mkdir(path.c_str(), 0777) == -1) + throw SysError("creating directory '%s'", path); + for (auto & attr : *v.attrs) + try { + if (attr.name == "." || attr.name == "..") + throw Error("invalid file name '%s'", attr.name); + recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name)); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name)); + throw; + } + } + else + throw TypeError("value at '%s' is not a string or an attribute set", pos); + }; + + recurse(*v, pos, *writeTo); + } + + else if (raw) { stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); - } else if (json) { + } + + else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); - } else { + } + + else { state->forceValueDeep(*v); logger->cout("%s", *v); } diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 43a765997..561ca53fb 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -16,3 +16,11 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x" + +rm -rf $TEST_ROOT/eval-out +nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' +[[ $(cat $TEST_ROOT/eval-out/x) = foobar ]] +[[ $(cat $TEST_ROOT/eval-out/y/z) = bla ]] + +rm -rf $TEST_ROOT/eval-out +(! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') From 72428e38d904a1db746f37c8727ab7b0fbb457bc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 23:05:28 +0100 Subject: [PATCH 152/477] Generate separate manpages for each nix subcommand --- .gitignore | 3 +- doc/manual/generate-manpage.nix | 50 +++++++++++++------- doc/manual/local.mk | 28 ++++++++--- doc/manual/src/{SUMMARY.md => SUMMARY.md.in} | 2 +- 4 files changed, 58 insertions(+), 25 deletions(-) rename doc/manual/src/{SUMMARY.md => SUMMARY.md.in} (99%) diff --git a/.gitignore b/.gitignore index c51582cf0..37aada307 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json -/doc/manual/src/command-ref/nix.md +/doc/manual/src/SUMMARY.md +/doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/expressions/builtins.md diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index db266750a..2801e96fa 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,34 +1,40 @@ +command: + with builtins; with import ./utils.nix; let showCommand = - { command, section, def }: - "${section} Name\n\n" + { command, def, filename }: + "# Name\n\n" + "`${command}` - ${def.description}\n\n" - + "${section} Synopsis\n\n" + + "# Synopsis\n\n" + showSynopsis { inherit command; args = def.args; } + + (if def.commands or {} != {} + then + "where *subcommand* is one of the following:\n\n" + # FIXME: group by category + + concatStrings (map (name: + "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n") + (attrNames def.commands)) + + "\n" + else "") + (if def ? doc - then "${section} Description\n\n" + def.doc + "\n\n" + then "# Description\n\n" + def.doc + "\n\n" else "") + (let s = showFlags def.flags; in if s != "" - then "${section} Flags\n\n${s}" + then "# Flags\n\n${s}" else "") + (if def.examples or [] != [] then - "${section} Examples\n\n" + "# Examples\n\n" + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else "") - + (if def.commands or [] != [] - then concatStrings ( - map (name: - "# Subcommand `${command} ${name}`\n\n" - + showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; }) - (attrNames def.commands)) else ""); + appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; + showFlags = flags: concatStrings (map (longName: @@ -48,8 +54,20 @@ let "`${command}` [*flags*...] ${concatStringsSep " " (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; + processCommand = { command, def, filename }: + [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ] + ++ concatMap + (name: processCommand { + filename = appendName filename name; + command = command + " " + name; + def = def.commands.${name}; + }) + (attrNames def.commands or {}); + in -command: - -showCommand { command = "nix"; section = "#"; def = command; } +let + manpages = processCommand { filename = "nix"; command = "nix"; def = command; }; + summary = concatStrings (map (manpage: " - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages); +in +(listToAttrs manpages) // { "SUMMARY.md" = summary; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index bb8b3b60a..b40fa4ed2 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -4,7 +4,7 @@ MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md) # Generate man pages. man-pages := $(foreach n, \ - nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 nix.1 \ + nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ nix-collect-garbage.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-hash.1 nix-copy-closure.1 \ @@ -22,7 +22,7 @@ dummy-env = env -i \ NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ NIX_STATE_DIR=/dummy -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @@ -42,13 +42,17 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(trace-gen) lowdown -sT man $^.tmp -o $@ @rm $^.tmp -$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix - $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli + $(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp @mv $@.tmp $@ +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix + @rm -rf $@ + $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' + $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -61,7 +65,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -71,7 +75,17 @@ $(d)/builtins.json: $(bindir)/nix # Generate the HTML manual. install: $(docdir)/manual/index.html -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/command-ref/nix.md $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md +# Generate 'nix' manpages. +install: $(d)/src/command-ref/new-cli + for i in doc/manual/src/command-ref/new-cli/*.md; do \ + name=$$(basename $$i .md); \ + if [[ $$name = SUMMARY ]]; then continue; fi; \ + printf "Title: %s\n\n" "$$name" > $$i.tmp; \ + cat $$i >> $$i.tmp; \ + lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \ + done + +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(trace-gen) mdbook build doc/manual -d $(docdir)/manual @cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md.in similarity index 99% rename from doc/manual/src/SUMMARY.md rename to doc/manual/src/SUMMARY.md.in index 8281f683f..b5ae34cfa 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md.in @@ -62,7 +62,7 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) - - [nix](command-ref/nix.md) +@manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) From e2efc63979a5485a96accaa1e49ffec65c353078 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 23:19:16 +0100 Subject: [PATCH 153/477] Put examples first in the manpages --- doc/manual/generate-manpage.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 2801e96fa..4a0b0290b 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -20,6 +20,11 @@ let (attrNames def.commands)) + "\n" else "") + + (if def.examples or [] != [] + then + "# Examples\n\n" + + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) + else "") + (if def ? doc then "# Description\n\n" + def.doc + "\n\n" else "") @@ -27,11 +32,7 @@ let if s != "" then "# Flags\n\n${s}" else "") - + (if def.examples or [] != [] - then - "# Examples\n\n" - + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else ""); + ; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; From b0de7b20160ba1910987c9165bc88689e6587c41 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 17:16:50 -0600 Subject: [PATCH 154/477] Check for rosetta 2 support before installing --- scripts/install.in | 5 +++++ 1 file changed, 5 insertions(+) mode change 100644 => 100755 scripts/install.in diff --git a/scripts/install.in b/scripts/install.in old mode 100644 new mode 100755 index 7c3b795cc..46d3817b9 --- a/scripts/install.in +++ b/scripts/install.in @@ -46,6 +46,11 @@ case "$(uname -s).$(uname -m)" in system=x86_64-darwin ;; Darwin.arm64) + # check for Rosetta 2 support + if ! [ -d /Library/Apple/usr/libexec/oah ]; then + oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" + fi + hash=@binaryTarball_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ # eventually maybe: arm64-darwin From addf9f4edeab8e8e2cb115dd98dc764cc1f19fcf Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 19:05:02 -0600 Subject: [PATCH 155/477] Call it aarch64-darwin instead of arm64-darwin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gnu-config standardized on aarch64 for machine name so host_cpu part of $system will always be aarch64. That means system will be aarch64-darwin too. uname however could report either “aarch64” (if gnu coreutils) or “arm64” (if apple’s uname). We should support both for compatiblity here. --- scripts/install.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 46d3817b9..9c8831d2b 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -45,7 +45,7 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_x86_64-darwin@ system=x86_64-darwin ;; - Darwin.arm64) + Darwin.arm64|Darwin.aarch64) # check for Rosetta 2 support if ! [ -d /Library/Apple/usr/libexec/oah ]; then oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" @@ -53,7 +53,7 @@ case "$(uname -s).$(uname -m)" in hash=@binaryTarball_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ - # eventually maybe: arm64-darwin + # eventually maybe: aarch64-darwin system=x86_64-darwin ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; From 94f359525ed913635eb21e8599d880db877bcc2a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 2 Dec 2020 19:14:34 -0600 Subject: [PATCH 156/477] Update config.guess for proper arm64 macOS detection This fixes results for arm64 macOS so config.guess now reports: aarch64-apple-darwin20.1.0 instead of arm-apple-darwin20.1.0 --- config/config.guess | 538 +++++++++---- config/config.sub | 1756 ++++++++++++++++++++++--------------------- 2 files changed, 1271 insertions(+), 1023 deletions(-) diff --git a/config/config.guess b/config/config.guess index d4fb3213e..699b3a10b 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-08-02' +timestamp='2020-11-19' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -27,12 +27,12 @@ timestamp='2018-08-02' # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess # # Please send patches to . -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ Usage: $0 [OPTION] @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -96,13 +96,14 @@ fi tmp= # shellcheck disable=SC2172 -trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15 -trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0 +trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 : "${TMPDIR=/tmp}" # shellcheck disable=SC2039 - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } @@ -130,16 +131,14 @@ if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi -UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown -UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown -UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown +UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown +UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown +UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - LIBC=gnu + LIBC=unknown set_cc_for_build cat <<-EOF > "$dummy.c" @@ -148,17 +147,29 @@ Linux|GNU|GNU/*) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc - #else + #elif defined(__GLIBC__) LIBC=gnu + #else + #include + /* First heuristic to detect musl libc. */ + #ifdef __DEFINED_va_list + LIBC=musl + #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')" - # If ldd exists, use it to detect musl libc. - if command -v ldd >/dev/null && \ - ldd --version 2>&1 | grep -q ^musl - then - LIBC=musl + # Second heuristic to detect musl libc. + if [ "$LIBC" = unknown ] && + command -v ldd >/dev/null && + ldd --version 2>&1 | grep -q ^musl; then + LIBC=musl + fi + + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + if [ "$LIBC" = unknown ]; then + LIBC=gnu fi ;; esac @@ -178,19 +189,20 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ "/sbin/$sysctl" 2>/dev/null || \ "/usr/sbin/$sysctl" 2>/dev/null || \ - echo unknown)` + echo unknown)) case "$UNAME_MACHINE_ARCH" in + aarch64eb) machine=aarch64_be-unknown ;; armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) - arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` - endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,') + endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p') machine="${arch}${endian}"-unknown ;; *) machine="$UNAME_MACHINE_ARCH"-unknown ;; @@ -221,7 +233,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in case "$UNAME_MACHINE_ARCH" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr") ;; esac # The OS release @@ -234,7 +246,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in release='-gnu' ;; *) - release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` + release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2) ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: @@ -243,15 +255,15 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//') echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//') echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" exit ;; *:LibertyBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//') echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" exit ;; *:MidnightBSD:*:*) @@ -263,6 +275,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:SolidBSD:*:*) echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; + *:OS108:*:*) + echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" + exit ;; macppc:MirBSD:*:*) echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; @@ -272,26 +287,29 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:Sortix:*:*) echo "$UNAME_MACHINE"-unknown-sortix exit ;; + *:Twizzler:*:*) + echo "$UNAME_MACHINE"-unknown-twizzler + exit ;; *:Redox:*:*) echo "$UNAME_MACHINE"-unknown-redox exit ;; mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}') ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}') ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on # OSF/1 and Tru64 systems produced since 1995. I hope that # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1) case "$ALPHA_CPU_TYPE" in "EV4 (21064)") UNAME_MACHINE=alpha ;; @@ -329,7 +347,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" + echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -363,7 +381,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "`(/bin/universe) 2>/dev/null`" = att ; then + if test "$( (/bin/universe) 2>/dev/null)" = att ; then echo pyramid-pyramid-sysv3 else echo pyramid-pyramid-bsd @@ -376,54 +394,59 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo sparc-icl-nx6 exit ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case `/usr/bin/uname -p` in + case $(/usr/bin/uname -p) in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" - case `isainfo -b` in - 32) - echo i386-pc-solaris2"$UNAME_REL" - ;; - 64) - echo x86_64-pc-solaris2"$UNAME_REL" - ;; - esac + set_cc_for_build + SUN_ARCH=i386 + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH=x86_64 + fi + fi + echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; sun4*:SunOS:*:*) - case "`/usr/bin/arch -k`" in + case "$(/usr/bin/arch -k)" in Series*|S4*) - UNAME_RELEASE=`uname -v` + UNAME_RELEASE=$(uname -v) ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" + echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')" exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) - UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null) test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case "`/bin/arch`" in + case "$(/bin/arch)" in sun3) echo m68k-sun-sunos"$UNAME_RELEASE" ;; @@ -503,8 +526,8 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in } EOF $CC_FOR_BUILD -o "$dummy" "$dummy.c" && - dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`"$dummy" "$dummyarg"` && + dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') && + SYSTEM_NAME=$("$dummy" "$dummyarg") && { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos"$UNAME_RELEASE" exit ;; @@ -531,11 +554,11 @@ EOF exit ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] + UNAME_PROCESSOR=$(/usr/bin/uname -p) + if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 then - if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ - [ "$TARGET_BINARY_INTERFACE"x = x ] + if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ + test "$TARGET_BINARY_INTERFACE"x = x then echo m88k-dg-dgux"$UNAME_RELEASE" else @@ -559,17 +582,17 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" + echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix exit ;; ia64:AIX:*:*) - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if test -x /usr/bin/oslevel ; then + IBM_REV=$(/usr/bin/oslevel) else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi @@ -589,7 +612,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") then echo "$SYSTEM_NAME" else @@ -602,15 +625,15 @@ EOF fi exit ;; *:AIX:*:[4567]) - IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }') if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/lslpp ] ; then - IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` + if test -x /usr/bin/lslpp ; then + IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/) else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi @@ -638,14 +661,14 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') case "$UNAME_MACHINE" in 9000/31?) HP_ARCH=m68000 ;; 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) - if [ -x /usr/bin/getconf ]; then - sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + if test -x /usr/bin/getconf; then + sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null) + sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null) case "$sc_cpu_version" in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 @@ -657,7 +680,7 @@ EOF esac ;; esac fi - if [ "$HP_ARCH" = "" ]; then + if test "$HP_ARCH" = ""; then set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" @@ -692,11 +715,11 @@ EOF exit (0); } EOF - (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy") test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ "$HP_ARCH" = hppa2.0w ] + if test "$HP_ARCH" = hppa2.0w then set_cc_for_build @@ -720,7 +743,7 @@ EOF echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) @@ -750,7 +773,7 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; @@ -770,7 +793,7 @@ EOF echo hppa1.0-hp-osf exit ;; i*86:OSF1:*:*) - if [ -x /usr/sbin/sysversion ] ; then + if test -x /usr/sbin/sysversion ; then echo "$UNAME_MACHINE"-unknown-osf1mk else echo "$UNAME_MACHINE"-unknown-osf1 @@ -819,14 +842,14 @@ EOF echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` + FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz) + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/') echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` + FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') + FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/') echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -838,26 +861,26 @@ EOF *:BSD/OS:*:*) echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" exit ;; - arm*:FreeBSD:*:*) - UNAME_PROCESSOR=`uname -p` + arm:FreeBSD:*:*) + UNAME_PROCESSOR=$(uname -p) set_cc_for_build if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi else - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf fi exit ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=$(/usr/bin/uname -p) case "$UNAME_PROCESSOR" in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac - echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; i*:CYGWIN*:*) echo "$UNAME_MACHINE"-pc-cygwin @@ -890,18 +913,18 @@ EOF echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-unknown-cygwin + echo x86_64-pc-cygwin exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" exit ;; *:GNU:*:*) # the GNU system - echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" + echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" + echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC" exit ;; *:Minix:*:*) echo "$UNAME_MACHINE"-unknown-minix @@ -914,7 +937,7 @@ EOF echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -981,22 +1004,50 @@ EOF exit ;; mips:Linux:*:* | mips64:Linux:*:*) set_cc_for_build + IS_GLIBC=0 + test x"${LIBC}" = xgnu && IS_GLIBC=1 sed 's/^ //' << EOF > "$dummy.c" #undef CPU - #undef ${UNAME_MACHINE} - #undef ${UNAME_MACHINE}el + #undef mips + #undef mipsel + #undef mips64 + #undef mips64el + #if ${IS_GLIBC} && defined(_ABI64) + LIBCABI=gnuabi64 + #else + #if ${IS_GLIBC} && defined(_ABIN32) + LIBCABI=gnuabin32 + #else + LIBCABI=${LIBC} + #endif + #endif + + #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa64r6 + #else + #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa32r6 + #else + #if defined(__mips64) + CPU=mips64 + #else + CPU=mips + #endif + #endif + #endif + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=${UNAME_MACHINE}el + MIPS_ENDIAN=el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=${UNAME_MACHINE} + MIPS_ENDIAN= #else - CPU= + MIPS_ENDIAN= #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" - test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } + eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')" + test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; mips64el:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -1015,7 +1066,7 @@ EOF exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level - case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; *) echo hppa-unknown-linux-"$LIBC" ;; @@ -1055,7 +1106,17 @@ EOF echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" + set_cc_for_build + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_X32 >/dev/null + then + LIBCABI="$LIBC"x32 + fi + fi + echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" exit ;; xtensa*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -1095,7 +1156,7 @@ EOF echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; i*86:*:4.*:*) - UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` + UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//') if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else @@ -1104,19 +1165,19 @@ EOF exit ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. - case `/bin/uname -X | grep "^Machine"` in + case $(/bin/uname -X | grep "^Machine") in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then - UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //')) (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ && UNAME_MACHINE=i586 @@ -1166,7 +1227,7 @@ EOF 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ @@ -1177,7 +1238,7 @@ EOF NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ @@ -1210,7 +1271,7 @@ EOF exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=`(uname -p) 2>/dev/null` + UNAME_MACHINE=$( (uname -p) 2>/dev/null) echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv @@ -1244,7 +1305,7 @@ EOF echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if [ -d /usr/nec ]; then + if test -d /usr/nec; then echo mips-nec-sysv"$UNAME_RELEASE" else echo mips-unknown-sysv"$UNAME_RELEASE" @@ -1292,44 +1353,48 @@ EOF *:Rhapsody:*:*) echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; + arm64:Darwin:*:*) + echo aarch64-apple-darwin"$UNAME_RELEASE" + exit ;; *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - set_cc_for_build - if test "$UNAME_PROCESSOR" = unknown ; then - UNAME_PROCESSOR=powerpc + UNAME_PROCESSOR=$(uname -p) + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + if command -v xcode-select > /dev/null 2> /dev/null && \ + ! xcode-select --print-path > /dev/null 2> /dev/null ; then + # Avoid executing cc if there is no toolchain installed as + # cc will be a stub that puts up a graphical alert + # prompting the user to install developer tools. + CC_FOR_BUILD=no_compiler_found + else + set_cc_for_build fi - if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then - if [ "$CC_FOR_BUILD" != no_compiler_found ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi + if test "$CC_FOR_BUILD" != no_compiler_found; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc fi elif test "$UNAME_PROCESSOR" = i386 ; then - # Avoid executing cc on OS X 10.9, as it ships with a stub - # that puts up a graphical alert prompting to install - # developer tools. Any system running Mac OS X 10.7 or - # later (Darwin 11 and later) is required to have a 64-bit - # processor. This is not true of the ARM version of Darwin - # that Apple uses in portable devices. - UNAME_PROCESSOR=x86_64 + # uname -m returns i386 or x86_64 + UNAME_PROCESSOR=$UNAME_MACHINE fi echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=`uname -p` + UNAME_PROCESSOR=$(uname -p) if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc @@ -1397,10 +1462,10 @@ EOF echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" exit ;; *:*VMS:*:*) - UNAME_MACHINE=`(uname -p) 2>/dev/null` + UNAME_MACHINE=$( (uname -p) 2>/dev/null) case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; @@ -1410,7 +1475,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" + echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')" exit ;; i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos @@ -1424,8 +1489,148 @@ EOF amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs exit ;; + *:Unleashed:*:*) + echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" + exit ;; esac +# No uname command or uname output not recognized. +set_cc_for_build +cat > "$dummy.c" < +#include +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#include +#if defined(_SIZE_T_) || defined(SIGLOST) +#include +#endif +#endif +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null); + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); +#endif + +#if defined (vax) +#if !defined (ultrix) +#include +#if defined (BSD) +#if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +#else +#if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#endif +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#else +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname un; + uname (&un); + printf ("vax-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("vax-dec-ultrix\n"); exit (0); +#endif +#endif +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname *un; + uname (&un); + printf ("mips-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("mips-dec-ultrix\n"); exit (0); +#endif +#endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. +test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } + echo "$0: unable to guess system type" >&2 case "$UNAME_MACHINE:$UNAME_SYSTEM" in @@ -1445,9 +1650,15 @@ This script (version $timestamp), has failed to recognize the operating system you are using. If your script is old, overwrite *all* copies of config.guess and config.sub with the latest versions from: - https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess + https://git.savannah.gnu.org/cgit/config.git/plain/config.guess and - https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub + https://git.savannah.gnu.org/cgit/config.git/plain/config.sub +EOF + +year=$(echo $timestamp | sed 's,-.*,,') +# shellcheck disable=SC2003 +if test "$(expr "$(date +%Y)" - "$year")" -lt 3 ; then + cat >&2 </dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` +uname -m = $( (uname -m) 2>/dev/null || echo unknown) +uname -r = $( (uname -r) 2>/dev/null || echo unknown) +uname -s = $( (uname -s) 2>/dev/null || echo unknown) +uname -v = $( (uname -v) 2>/dev/null || echo unknown) -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null` +/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null) +/bin/uname -X = $( (/bin/uname -X) 2>/dev/null) -hostinfo = `(hostinfo) 2>/dev/null` -/bin/universe = `(/bin/universe) 2>/dev/null` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` -/bin/arch = `(/bin/arch) 2>/dev/null` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` +hostinfo = $( (hostinfo) 2>/dev/null) +/bin/universe = $( (/bin/universe) 2>/dev/null) +/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null) +/bin/arch = $( (/bin/arch) 2>/dev/null) +/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null) +/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null) UNAME_MACHINE = "$UNAME_MACHINE" UNAME_RELEASE = "$UNAME_RELEASE" UNAME_SYSTEM = "$UNAME_SYSTEM" UNAME_VERSION = "$UNAME_VERSION" EOF +fi exit 1 diff --git a/config/config.sub b/config/config.sub index c19e67180..19c9553b1 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-08-13' +timestamp='2020-12-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -33,7 +33,7 @@ timestamp='2018-08-13' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub +# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -50,7 +50,7 @@ timestamp='2018-08-13' # CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM # It is wrong to echo any other type of specification. -me=`echo "$0" | sed -e 's,.*/,,'` +me=$(echo "$0" | sed -e 's,.*/,,') usage="\ Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS @@ -67,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -89,7 +89,7 @@ while test $# -gt 0 ; do - ) # Use stdin as input. break ;; -* ) - echo "$me: invalid option $1$help" + echo "$me: invalid option $1$help" >&2 exit 1 ;; *local*) @@ -111,7 +111,8 @@ case $# in esac # Split fields of configuration type -IFS="-" read -r field1 field2 field3 field4 <&2 - exit 1 + # Recognize the canonical CPU types that are allowed with any + # company name. + case $cpu in + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ + | alphapca5[67] | alpha64pca5[67] \ + | am33_2.0 \ + | amdgcn \ + | arc | arceb \ + | arm | arm[lb]e | arme[lb] | armv* \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin | bpf | bs2000 \ + | c[123]* | c30 | [cjt]90 | c4x \ + | c8051 | clipper | craynv | csky | cydra \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | elxsi | epiphany \ + | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | h8300 | h8500 \ + | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i*86 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle \ + | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ + | m88110 | m88k | maxq | mb | mcore | mep | metag \ + | microblaze | microblazeel \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64eb | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mmix \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | none | np1 | ns16k | ns32k | nvptx \ + | open8 \ + | or1k* \ + | or32 \ + | orion \ + | picochip \ + | pdp10 | pdp11 | pj | pjl | pn | power \ + | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | romp | rs6000 | rx \ + | s390 | s390x \ + | score \ + | sh | shl \ + | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ + | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | spu \ + | tahoe \ + | thumbv7* \ + | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tron \ + | ubicom32 \ + | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | vax \ + | visium \ + | w65 \ + | wasm32 | wasm64 \ + | we32k \ + | x86 | x86_64 | xc16x | xgate | xps100 \ + | xstormy16 | xtensa* \ + | ymp \ + | z8k | z80) + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + exit 1 + ;; + esac ;; esac # Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` +case $vendor in + digital*) + vendor=dec ;; - *-commodore*) - basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` + commodore*) + vendor=cbm ;; *) ;; @@ -1296,8 +1279,47 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x$os != x ] +if test x$basic_os != x then + +# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# set os. +case $basic_os in + gnu/linux*) + kernel=linux + os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|') + ;; + os2-emx) + kernel=os2 + os=$(echo $basic_os | sed -e 's|os2-emx|emx|') + ;; + nto-qnx*) + kernel=nto + os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|') + ;; + *-*) + # shellcheck disable=SC2162 + IFS="-" read kernel os <&2 - exit 1 + # No normalization, but not necessarily accepted, that comes below. ;; esac + else # Here we handle the default operating systems that come with various machines. @@ -1551,7 +1498,8 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. -case $basic_machine in +kernel= +case $cpu-$vendor in score-*) os=elf ;; @@ -1562,7 +1510,8 @@ case $basic_machine in os=riscix1.2 ;; arm*-rebel) - os=linux + kernel=linux + os=gnu ;; arm*-semi) os=aout @@ -1728,86 +1677,173 @@ case $basic_machine in os=none ;; esac + fi -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) - case $os in - riscix*) - vendor=acorn - ;; - sunos*) - vendor=sun - ;; - cnk*|-aix*) - vendor=ibm - ;; - beos*) - vendor=be - ;; - hpux*) - vendor=hp - ;; - mpeix*) - vendor=hp - ;; - hiux*) - vendor=hitachi - ;; - unos*) - vendor=crds - ;; - dgux*) - vendor=dg - ;; - luna*) - vendor=omron - ;; - genix*) - vendor=ns - ;; - clix*) - vendor=intergraph - ;; - mvs* | opened*) - vendor=ibm - ;; - os400*) - vendor=ibm - ;; - ptx*) - vendor=sequent - ;; - tpf*) - vendor=ibm - ;; - vxsim* | vxworks* | windiss*) - vendor=wrs - ;; - aux*) - vendor=apple - ;; - hms*) - vendor=hitachi - ;; - mpw* | macos*) - vendor=apple - ;; - *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) - vendor=atari - ;; - vos*) - vendor=stratus - ;; - esac - basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` +# Now, validate our (potentially fixed-up) OS. +case $os in + # Sometimes we do "kernel-abi", so those need to count as OSes. + musl* | newlib* | uclibc*) + ;; + # Likewise for "kernel-libc" + eabi | eabihf | gnueabi | gnueabihf) + ;; + # Now accept the basic system types. + # The portable systems comes first. + # Each alternative MUST end in a * to match a version number. + gnu* | android* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]* \ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ + | hiux* | abug | nacl* | netware* | windows* \ + | os9* | macos* | osx* | ios* \ + | mpw* | magic* | mmixware* | mon960* | lnews* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* | twizzler* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | mirbsd* | netbsd* | dicos* | openedition* | ose* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \ + | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | mint* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* | morphos* \ + | scout* | superux* | sysv* | rtmk* | tpf* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx*) + ;; + # This one is extra strict with allowed versions + sco3.2v2 | sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + none) + ;; + *) + echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + exit 1 ;; esac -echo "$basic_machine-$os" +# As a final step for OS-related things, validate the OS-kernel combination +# (given a valid OS), if there is a kernel. +case $kernel-$os in + linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) + ;; + uclinux-uclibc* ) + ;; + -dietlibc* | -newlib* | -musl* | -uclibc* ) + # These are just libc implementations, not actual OSes, and thus + # require a kernel. + echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + exit 1 + ;; + kfreebsd*-gnu* | kopensolaris*-gnu*) + ;; + nto-qnx*) + ;; + os2-emx) + ;; + *-eabi* | *-gnueabi*) + ;; + -*) + # Blank kernel with real OS is always fine. + ;; + *-*) + echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + exit 1 + ;; +esac + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +case $vendor in + unknown) + case $cpu-$os in + *-riscix*) + vendor=acorn + ;; + *-sunos*) + vendor=sun + ;; + *-cnk* | *-aix*) + vendor=ibm + ;; + *-beos*) + vendor=be + ;; + *-hpux*) + vendor=hp + ;; + *-mpeix*) + vendor=hp + ;; + *-hiux*) + vendor=hitachi + ;; + *-unos*) + vendor=crds + ;; + *-dgux*) + vendor=dg + ;; + *-luna*) + vendor=omron + ;; + *-genix*) + vendor=ns + ;; + *-clix*) + vendor=intergraph + ;; + *-mvs* | *-opened*) + vendor=ibm + ;; + *-os400*) + vendor=ibm + ;; + s390-* | s390x-*) + vendor=ibm + ;; + *-ptx*) + vendor=sequent + ;; + *-tpf*) + vendor=ibm + ;; + *-vxsim* | *-vxworks* | *-windiss*) + vendor=wrs + ;; + *-aux*) + vendor=apple + ;; + *-hms*) + vendor=hitachi + ;; + *-mpw* | *-macos*) + vendor=apple + ;; + *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) + vendor=atari + ;; + *-vos*) + vendor=stratus + ;; + esac + ;; +esac + +echo "$cpu-$vendor-${kernel:+$kernel-}$os" exit # Local variables: From 0afab668fa3f20f091dec0a31fc0b0fbaac2afde Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 3 Dec 2020 13:19:20 +0100 Subject: [PATCH 157/477] Don't fail early when -j0 is passed If the build closure contains some CA derivations, then we can't know ahead-of-time that we won't build anything as early-cutoff might come-in at a laster stage --- src/libstore/build/worker.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 1f8999a4b..6c96a93bd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -223,11 +223,6 @@ void Worker::run(const Goals & _topGoals) uint64_t downloadSize, narSize; store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); - debug("entered goal loop"); while (1) { From 8ad72b1f1c272d01d5d67b21553fc261c8df5302 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 3 Dec 2020 13:20:53 +0100 Subject: [PATCH 158/477] Properly test early cutoff with CA derivations Build things with a different seed each time to make sure that it works despite the different drvs --- tests/content-addressed.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 52f7529b5..03eff549c 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -7,30 +7,32 @@ nix --experimental-features 'nix-command ca-derivations' show-derivation --deriv buildAttr () { local derivationPath=$1 - shift - local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + local seedValue=$2 + shift; shift + local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") args+=("$@") nix-build "${args[@]}" } testRemoteCache () { clearCache - local outPath=$(buildAttr dependentNonCA) + local outPath=$(buildAttr dependentNonCA 1) nix copy --to file://$cacheDir $outPath clearStore - buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") + buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") } testDeterministicCA () { - [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]] + [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } testCutoffFor () { local out1 out2 - out1=$(buildAttr $1) + out1=$(buildAttr $1 1) # The seed only changes the root derivation, and not it's output, so the # dependent derivations should only need to be built once. - out2=$(buildAttr $1 -j0) + buildAttr rootCA 2 + out2=$(buildAttr $1 2 -j0) test "$out1" == "$out2" } From c3c858ac6d0c75bd95bd6913276ef20cf2495e96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 13:55:19 +0100 Subject: [PATCH 159/477] Make doc() return arbitrary Markdown rather than the contents of the "Description" section Thus we can return the examples section (and any other sections) from doc() and don't need examples() anymore. --- doc/manual/generate-manpage.nix | 2 +- src/nix/add-to-store.cc | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 4a0b0290b..fbd7f3e7d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -26,7 +26,7 @@ let + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) else "") + (if def ? doc - then "# Description\n\n" + def.doc + "\n\n" + then def.doc + "\n\n" else "") + (let s = showFlags def.flags; in if s != "" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index df51e72d5..022f0ea85 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -39,15 +39,29 @@ struct CmdAddToStore : MixDryRun, StoreCommand std::string doc() override { return R"( + # Description + Copy the file or directory *path* to the Nix store, and print the resulting store path on standard output. - )"; - } - Examples examples() override - { - return { - }; + > **Warning** + > + > The resulting store path is not registered as a garbage + > collector root, so it could be deleted before you have a + > chance to register it. + + # Examples + + Add a regular file to the store: + + ```console + # echo foo > bar + # nix add-to-store --flat ./bar + /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + foo + ``` + )"; } Category category() override { return catUtility; } From 1b0ca3866bae7185e628fda956634c52a5da7a15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 14:23:18 +0100 Subject: [PATCH 160/477] nix add-to-store: Move markdown docs into a separate file --- src/nix/add-to-store.cc | 27 +++------------------------ src/nix/add-to-store.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 src/nix/add-to-store.md diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 022f0ea85..a2721431e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -38,30 +38,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand std::string doc() override { - return R"( - # Description - - Copy the file or directory *path* to the Nix store, and - print the resulting store path on standard output. - - > **Warning** - > - > The resulting store path is not registered as a garbage - > collector root, so it could be deleted before you have a - > chance to register it. - - # Examples - - Add a regular file to the store: - - ```console - # echo foo > bar - # nix add-to-store --flat ./bar - /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - foo - ``` - )"; + return + #include "add-to-store.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/add-to-store.md b/src/nix/add-to-store.md new file mode 100644 index 000000000..593ad67ad --- /dev/null +++ b/src/nix/add-to-store.md @@ -0,0 +1,28 @@ +R""( + +# Description + +Copy the file or directory *path* to the Nix store, and +print the resulting store path on standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a regular file to the store: + +```console +# echo foo > bar + +# nix add-to-store --flat ./bar +/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + +# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar +foo +``` + +)"" From 8ad2c9c4b97f291982598e34530122612c580b83 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 14:35:42 +0100 Subject: [PATCH 161/477] Remove 'dist' target We're not producing source tarballs anymore so this has been bitrotting. --- doc/manual/local.mk | 2 -- local.mk | 6 ------ mk/dist.mk | 17 ----------------- mk/lib.mk | 4 ---- mk/libraries.mk | 1 - mk/programs.mk | 1 - nix-rust/local.mk | 2 -- src/libexpr/local.mk | 2 -- 8 files changed, 35 deletions(-) delete mode 100644 mk/dist.mk diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b40fa4ed2..ee5b328de 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -13,8 +13,6 @@ man-pages := $(foreach n, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 -dist-files += $(man-pages) - # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. dummy-env = env -i \ HOME=/dummy \ diff --git a/local.mk b/local.mk index fe314b902..6a7074e8e 100644 --- a/local.mk +++ b/local.mk @@ -1,9 +1,3 @@ -ifeq ($(MAKECMDGOALS), dist) - dist-files += $(shell cat .dist-files) -endif - -dist-files += configure config.h.in perl/configure - clean-files += Makefile.config GLOBAL_CXXFLAGS += -Wno-deprecated-declarations diff --git a/mk/dist.mk b/mk/dist.mk deleted file mode 100644 index 794b27771..000000000 --- a/mk/dist.mk +++ /dev/null @@ -1,17 +0,0 @@ -ifdef PACKAGE_NAME - -dist-name = $(PACKAGE_NAME)-$(PACKAGE_VERSION) - -dist: $(dist-name).tar.bz2 $(dist-name).tar.xz - -$(dist-name).tar.bz2: $(dist-files) - $(trace-gen) tar cfj $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' - -$(dist-name).tar.xz: $(dist-files) - $(trace-gen) tar cfJ $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' - -clean-files += $(dist-name).tar.bz2 $(dist-name).tar.xz - -print-top-help += echo " dist: Generate a source distribution"; - -endif diff --git a/mk/lib.mk b/mk/lib.mk index 1da51d879..a09ebaa97 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,7 +10,6 @@ bin-scripts := noinst-scripts := man-pages := install-tests := -dist-files := OS = $(shell uname -s) @@ -112,9 +111,6 @@ $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) -include mk/dist.mk - - .PHONY: default all man help all: $(programs-list) $(libs-list) $(jars-list) $(man-pages) diff --git a/mk/libraries.mk b/mk/libraries.mk index e6ef2e3ec..7c0e4f100 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -159,5 +159,4 @@ define build-library libs-list += $$($(1)_PATH) endif clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) - dist-files += $$(_srcs) endef diff --git a/mk/programs.mk b/mk/programs.mk index 3fa9685c3..d0cf5baf0 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -79,7 +79,6 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) - dist-files += $$(_srcs) # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN diff --git a/nix-rust/local.mk b/nix-rust/local.mk index e4bfde31b..50db4783c 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -30,8 +30,6 @@ ifeq ($(OS), Darwin) install_name_tool -id $@ $@ endif -dist-files += $(d)/vendor - clean: clean-rust clean-rust: diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a5422169d..519da33f7 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -35,8 +35,6 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh - $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ From b2d6c6161e61c47f4d8a3faa5799160d100f4e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 18:44:43 +0200 Subject: [PATCH 162/477] Move 'nix hash-*' and 'nix to-*' to 'nix hash' From the 'nix' UX review. --- src/nix/hash.cc | 52 +++++++++++++++++++++++++++++++++------------- tests/brotli.sh | 4 ++-- tests/fetchurl.sh | 8 +++---- tests/hash.sh | 10 ++++----- tests/pure-eval.sh | 2 +- tests/tarball.sh | 2 +- 6 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 7f3d5717a..101b67e6a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdHash : Command +struct CmdHashBase : Command { FileIngestionMethod mode; Base base = SRI; @@ -17,7 +17,7 @@ struct CmdHash : Command std::vector paths; std::optional modulus; - CmdHash(FileIngestionMethod mode) : mode(mode) + CmdHashBase(FileIngestionMethod mode) : mode(mode) { mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); @@ -51,8 +51,6 @@ struct CmdHash : Command return d; } - Category category() override { return catUtility; } - void run() override { for (auto path : paths) { @@ -79,9 +77,6 @@ struct CmdHash : Command } }; -static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); -static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); - struct CmdToBase : Command { Base base; @@ -103,8 +98,6 @@ struct CmdToBase : Command "SRI"); } - Category category() override { return catUtility; } - void run() override { for (auto s : args) @@ -112,10 +105,41 @@ struct CmdToBase : Command } }; -static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref(SRI); }); +struct CmdHash : NixMultiCommand +{ + CmdHash() + : MultiCommand({ + {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, + {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, + {"to-base16", []() { return make_ref(Base16); }}, + {"to-base32", []() { return make_ref(Base32); }}, + {"to-base64", []() { return make_ref(Base64); }}, + {"to-sri", []() { return make_ref(SRI); }}, + }) + { } + + std::string description() override + { + return "compute and convert cryptographic hashes"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix hash' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -149,7 +173,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; diff --git a/tests/brotli.sh b/tests/brotli.sh index a3c6e55a8..dc9bbdb66 100644 --- a/tests/brotli.sh +++ b/tests/brotli.sh @@ -9,13 +9,13 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to $cacheURI $outPath -HASH=$(nix hash-path $outPath) +HASH=$(nix hash path $outPath) clearStore clearCacheCache nix copy --from $cacheURI $outPath --no-check-sigs -HASH2=$(nix hash-path $outPath) +HASH2=$(nix hash path $outPath) [[ $HASH = $HASH2 ]] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 0f2044342..89ff08d4d 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -12,7 +12,7 @@ cmp $outPath fetchurl.sh # Now using a base-64 hash. clearStore -hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) +hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh) outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) @@ -21,7 +21,7 @@ cmp $outPath fetchurl.sh # Now using an SRI hash. clearStore -hash=$(nix hash-file ./fetchurl.sh) +hash=$(nix hash file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] @@ -34,14 +34,14 @@ clearStore other_store=file://$TEST_ROOT/other_store?store=/fnord/store -hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) +hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ --no-out-link --substituters $other_store # Test unpacking a NAR. diff --git a/tests/hash.sh b/tests/hash.sh index 4cfc97901..e5f75e2cf 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -2,7 +2,7 @@ source common.sh try () { printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) + hash=$(nix hash file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) if test "$hash" != "$3"; then echo "hash $1, expected $3, got $hash" exit 1 @@ -69,17 +69,17 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion. try3() { - h64=$(nix to-base64 --type "$1" "$2") + h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] - sri=$(nix to-sri --type "$1" "$2") + sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] - h16=$(nix to-base16 --type "$1" "$h64") + h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] - h16=$(nix to-base16 "$sri") + h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 561ca53fb..c994fbb98 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -15,7 +15,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") -nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x" +nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x" rm -rf $TEST_ROOT/eval-out nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' diff --git a/tests/tarball.sh b/tests/tarball.sh index fe65a22e4..d53ec8cd9 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -10,7 +10,7 @@ mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ -hash=$(nix hash-path $tarroot) +hash=$(nix hash path $tarroot) test_tarball() { local ext="$1" From 5781f45c46d7f6c55f87528ab1e754d070bc99ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 17:55:27 +0100 Subject: [PATCH 163/477] Allow registering subcommands of subcommands --- src/nix/command.cc | 16 +++++++++++++++- src/nix/command.hh | 13 +++++++++++-- src/nix/main.cc | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 9a38c77f1..596217775 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -11,7 +11,21 @@ extern char * * environ __attribute__((weak)); namespace nix { -Commands * RegisterCommand::commands = nullptr; +RegisterCommand::Commands * RegisterCommand::commands = nullptr; + +nix::Commands RegisterCommand::getCommandsFor(const std::vector & prefix) +{ + nix::Commands res; + for (auto & [name, command] : *RegisterCommand::commands) + if (name.size() == prefix.size() + 1) { + bool equal = true; + for (size_t i = 0; i < prefix.size(); ++i) + if (name[i] != prefix[i]) equal = false; + if (equal) + res.insert_or_assign(name[prefix.size()], command); + } + return res; +} void NixMultiCommand::printHelp(const string & programName, std::ostream & out) { diff --git a/src/nix/command.hh b/src/nix/command.hh index d60c8aeb6..6882db195 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -176,20 +176,29 @@ struct StorePathCommand : public InstallablesCommand /* A helper class for registering commands globally. */ struct RegisterCommand { + typedef std::map, std::function()>> Commands; static Commands * commands; - RegisterCommand(const std::string & name, + RegisterCommand(std::vector && name, std::function()> command) { if (!commands) commands = new Commands; commands->emplace(name, command); } + + static nix::Commands getCommandsFor(const std::vector & prefix); }; template static RegisterCommand registerCommand(const std::string & name) { - return RegisterCommand(name, [](){ return make_ref(); }); + return RegisterCommand({name}, [](){ return make_ref(); }); +} + +template +static RegisterCommand registerCommand2(std::vector && name) +{ + return RegisterCommand(std::move(name), [](){ return make_ref(); }); } Buildables build(ref store, Realise mode, diff --git a/src/nix/main.cc b/src/nix/main.cc index a75f8ae65..6d8c66406 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,7 +59,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs bool useNet = true; bool refresh = false; - NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[Command::catDefault] = "Main commands"; From 79c1967ded92574129c6a20116ef205a9c747bac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 18:06:46 +0100 Subject: [PATCH 164/477] Introduce 'nix store' command --- src/nix/store.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/nix/store.cc diff --git a/src/nix/store.cc b/src/nix/store.cc new file mode 100644 index 000000000..e91bcc503 --- /dev/null +++ b/src/nix/store.cc @@ -0,0 +1,31 @@ +#include "command.hh" + +using namespace nix; + +struct CmdStore : virtual NixMultiCommand +{ + CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + { } + + std::string description() override + { + return "manipulate a Nix store"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix store' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdStore = registerCommand("store"); From ef583303f0720d8bc9d6351cd769f92d5dd678f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:42:24 +0200 Subject: [PATCH 165/477] Move NAR-related commands to 'nix nar' --- src/nix/cat.cc | 6 ++---- src/nix/ls.cc | 8 +++----- src/nix/nar.cc | 31 +++++++++++++++++++++++++++++++ tests/binary-cache.sh | 6 +++--- tests/linux-sandbox.sh | 4 ++-- tests/nar-access.sh | 26 +++++++++++++------------- 6 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 src/nix/nar.cc diff --git a/src/nix/cat.cc b/src/nix/cat.cc index eef172cfc..4fa1c9491 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -64,13 +64,11 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } - Category category() override { return catUtility; } - void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static auto rCmdCatStore = registerCommand("cat-store"); -static auto rCmdCatNar = registerCommand("cat-nar"); +static auto rCmdCatStore = registerCommand2({"store", "cat"}); +static auto rCmdCatNar = registerCommand2({"nar", "cat"}); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index f39fdb2fd..d5fec4d84 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -134,7 +134,7 @@ struct CmdLsNar : Command, MixLs return { Example{ "To list a specific file in a NAR:", - "nix ls-nar -l hello.nar /bin/hello" + "nix nar ls -l hello.nar /bin/hello" }, }; } @@ -144,13 +144,11 @@ struct CmdLsNar : Command, MixLs return "show information about a path inside a NAR file"; } - Category category() override { return catUtility; } - void run() override { list(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static auto rCmdLsStore = registerCommand("ls-store"); -static auto rCmdLsNar = registerCommand("ls-nar"); +static auto rCmdLsStore = registerCommand2({"store", "ls"}); +static auto rCmdLsNar = registerCommand2({"nar", "ls"}); diff --git a/src/nix/nar.cc b/src/nix/nar.cc new file mode 100644 index 000000000..e239ce96a --- /dev/null +++ b/src/nix/nar.cc @@ -0,0 +1,31 @@ +#include "command.hh" + +using namespace nix; + +struct CmdNar : NixMultiCommand +{ + CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + { } + + std::string description() override + { + return "query the contents of NAR files"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix nar' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto rCmdNar = registerCommand("nar"); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index e3b3982fe..8cb17caf8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -196,13 +196,13 @@ narCache=$TEST_ROOT/nar-cache rm -rf $narCache mkdir $narCache -[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] -(! nix cat-store --store file://$cacheDir $outPath/foobar) +(! nix store cat --store file://$cacheDir $outPath/foobar) # Test NAR listing generation. diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 16abd974c..70a90a907 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -22,9 +22,9 @@ outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) nix path-info -r $outPath | grep input-2 -nix ls-store -R -l $outPath | grep foobar +nix store ls -R -l $outPath | grep foobar -nix cat-store $outPath/foobar | grep FOOBAR +nix store cat $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store diff --git a/tests/nar-access.sh b/tests/nar-access.sh index 88b997ca6..dcc2e8a36 100644 --- a/tests/nar-access.sh +++ b/tests/nar-access.sh @@ -9,45 +9,45 @@ cd "$TEST_ROOT" narFile="$TEST_ROOT/path.nar" nix-store --dump $storePath > $narFile -# Check that find and ls-nar match. +# Check that find and nar ls match. ( cd $storePath; find . | sort ) > files.find -nix ls-nar -R -d $narFile "" | sort > files.ls-nar +nix nar ls -R -d $narFile "" | sort > files.ls-nar diff -u files.find files.ls-nar # Check that file contents of data match. -nix cat-nar $narFile /foo/data > data.cat-nar +nix nar cat $narFile /foo/data > data.cat-nar diff -u data.cat-nar $storePath/foo/data # Check that file contents of baz match. -nix cat-nar $narFile /foo/baz > baz.cat-nar +nix nar cat $narFile /foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz -nix cat-store $storePath/foo/baz > baz.cat-nar +nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz # Test --json. diff -u \ - <(nix ls-nar --json $narFile / | jq -S) \ + <(nix nar ls --json $narFile / | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix ls-nar --json -R $narFile /foo | jq -S) \ + <(nix nar ls --json -R $narFile /foo | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) diff -u \ - <(nix ls-nar --json -R $narFile /foo/bar | jq -S) \ + <(nix nar ls --json -R $narFile /foo/bar | jq -S) \ <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) diff -u \ - <(nix ls-store --json $storePath | jq -S) \ + <(nix store ls --json $storePath | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix ls-store --json -R $storePath/foo | jq -S) \ + <(nix store ls --json -R $storePath/foo | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) diff -u \ - <(nix ls-store --json -R $storePath/foo/bar| jq -S) \ + <(nix store ls --json -R $storePath/foo/bar| jq -S) \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' -nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist' +nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. if nix-store --dump $storePath >/dev/full ; then From 0c15ae5d4b3366a14bca885e02599a941a334920 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:44:31 +0200 Subject: [PATCH 166/477] Add FIXME --- src/nix/show-derivation.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 6d4f295d7..8e1a58ac2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -1,4 +1,5 @@ // FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? #include "command.hh" #include "common-args.hh" From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: [PATCH 167/477] Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: [PATCH 168/477] Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); From af373c2ece2c14ac652313a6f370dc344c85f86e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 22:45:44 +0100 Subject: [PATCH 169/477] Add deprecated aliases for renamed commands --- src/libutil/args.cc | 5 +---- src/libutil/args.hh | 5 +++-- src/nix/main.cc | 30 +++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8bd9c8aeb..61f9503ec 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -86,6 +86,7 @@ void Args::parseCmdline(const Strings & _cmdline) throw UsageError("unrecognised flag '%1%'", arg); } else { + pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) pendingArgs.clear(); @@ -390,10 +391,6 @@ MultiCommand::MultiCommand(const Commands & commands) .optional = true, .handler = {[=](std::string s) { assert(!command); - if (auto alias = get(deprecatedAliases, s)) { - warn("'%s' is a deprecated alias for '%s'", s, *alias); - s = *alias; - } if (auto prefix = needsCompletion(s)) { for (auto & [name, command] : commands) if (hasPrefix(name, *prefix)) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 26f1bc11b..8069fd70f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -115,6 +115,9 @@ protected: virtual bool processArgs(const Strings & args, bool finish); + virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) + { return pos; } + std::set hiddenCategories; public: @@ -257,8 +260,6 @@ public: std::map categories; - std::map deprecatedAliases; - // Selected command, if any. std::optional>> command; diff --git a/src/nix/main.cc b/src/nix/main.cc index 6d8c66406..94f4cad3c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -112,8 +112,36 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "consider all previously downloaded files out-of-date", .handler = {[&]() { refresh = true; }}, }); + } - deprecatedAliases.insert({"dev-shell", "develop"}); + std::map> aliases = { + {"dev-shell", {"develop"}}, + {"hash-file", {"hash", "file"}}, + {"hash-path", {"hash", "path"}}, + {"to-base16", {"hash", "to-base16"}}, + {"to-base32", {"hash", "to-base32"}}, + {"to-base64", {"hash", "to-base64"}}, + {"ls-nar", {"nar", "ls"}}, + {"ls-store", {"store", "ls"}}, + {"cat-nar", {"nar", "cat"}}, + {"cat-store", {"store", "cat"}}, + }; + + bool aliasUsed = false; + + Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override + { + if (aliasUsed || command || pos == args.end()) return pos; + auto arg = *pos; + auto i = aliases.find(arg); + if (i == aliases.end()) return pos; + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", i->second)); + pos = args.erase(pos); + for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) + pos = args.insert(pos, *j); + aliasUsed = true; + return pos; } void printFlags(std::ostream & out) override From a1cd805cba7a4408e75779bc4099f92e81fd6ac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 21:04:26 +0200 Subject: [PATCH 170/477] Add 'nix nar dump-path' This only differs from 'nix store dump-path' in that the path doesn't need to be a store path. --- src/nix/dump-path.cc | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/nix/main.cc | 1 + 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 6fd197531..4b225ae9f 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "archive.hh" using namespace nix; @@ -7,7 +8,7 @@ struct CmdDumpPath : StorePathCommand { std::string description() override { - return "dump a store path to stdout (in NAR format)"; + return "serialise a store path to stdout in NAR format"; } Examples examples() override @@ -30,4 +31,43 @@ struct CmdDumpPath : StorePathCommand } }; -static auto rDumpPath = registerCommand("dump-path"); + +static auto rDumpPath = registerCommand2({"store", "dump-path"}); + +struct CmdDumpPath2 : Command +{ + Path path; + + CmdDumpPath2() + { + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); + } + + std::string description() override + { + return "serialise a path to stdout in NAR format"; + } + + Examples examples() override + { + return { + Example{ + "To serialise directory 'foo' as a NAR:", + "nix nar dump-path ./foo" + }, + }; + } + + void run() override + { + FdSink sink(STDOUT_FILENO); + dumpPath(path, sink); + sink.flush(); + } +}; + +static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/main.cc b/src/nix/main.cc index 94f4cad3c..0002be291 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -125,6 +125,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"ls-store", {"store", "ls"}}, {"cat-nar", {"nar", "cat"}}, {"cat-store", {"store", "cat"}}, + {"dump-path", {"store", "dump-path"}}, }; bool aliasUsed = false; From ea2062a2d9144d78588675950fc04756f0d200a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:38:56 +0200 Subject: [PATCH 171/477] Move most store-related commands to 'nix store' --- src/nix/add-to-store.cc | 4 +-- src/nix/cat.cc | 2 -- src/nix/diff-closures.cc | 6 ++-- src/nix/dump-path.cc | 5 +-- src/nix/ls.cc | 4 +-- src/nix/main.cc | 18 ++++++++--- src/nix/make-content-addressable.cc | 8 ++--- src/nix/optimise-store.cc | 6 ++-- src/nix/ping-store.cc | 6 ++-- src/nix/sigs.cc | 8 ++--- src/nix/verify.cc | 8 ++--- tests/binary-cache.sh | 2 +- tests/fetchurl.sh | 2 +- tests/gc-auto.sh | 6 ++-- tests/recursive.sh | 4 +-- tests/signing.sh | 48 ++++++++++++++--------------- tests/ssh-relay.sh | 2 +- 17 files changed, 62 insertions(+), 77 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index a2721431e..66822b0ff 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,8 +43,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand ; } - Category category() override { return catUtility; } - void run(ref store) override { if (!namePart) namePart = baseNameOf(path); @@ -80,4 +78,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto rCmdAddToStore = registerCommand("add-to-store"); +static auto rCmdAddToStore = registerCommand2({"store", "add-path"}); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 4fa1c9491..2ecffc9a5 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -37,8 +37,6 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } - Category category() override { return catUtility; } - void run(ref store) override { cat(store->getFSAccessor()); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 30e7b20e1..f72b5eff7 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -121,14 +121,12 @@ struct CmdDiffClosures : SourceExprCommand return "show what packages and versions were added and removed between two closures"; } - Category category() override { return catSecondary; } - Examples examples() override { return { { "To show what got added and removed between two versions of the NixOS system profile:", - "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", + "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", }, }; } @@ -143,4 +141,4 @@ struct CmdDiffClosures : SourceExprCommand } }; -static auto rCmdDiffClosures = registerCommand("diff-closures"); +static auto rCmdDiffClosures = registerCommand2({"store", "diff-closures"}); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 4b225ae9f..256db64a9 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -16,13 +16,11 @@ struct CmdDumpPath : StorePathCommand return { Example{ "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" }, }; } - Category category() override { return catUtility; } - void run(ref store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); @@ -31,7 +29,6 @@ struct CmdDumpPath : StorePathCommand } }; - static auto rDumpPath = registerCommand2({"store", "dump-path"}); struct CmdDumpPath2 : Command diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d5fec4d84..1f5ed6913 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -97,7 +97,7 @@ struct CmdLsStore : StoreCommand, MixLs return { Example{ "To list the contents of a store path in a binary cache:", - "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" }, }; } @@ -107,8 +107,6 @@ struct CmdLsStore : StoreCommand, MixLs return "show information about a path in the Nix store"; } - Category category() override { return catUtility; } - void run(ref store) override { list(store->getFSAccessor()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 0002be291..fb3bffeaf 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -115,17 +115,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs } std::map> aliases = { + {"add-to-store", {"store", "add-path"}}, + {"cat-nar", {"nar", "cat"}}, + {"cat-store", {"store", "cat"}}, + {"copy-sigs", {"store", "copy-sigs"}}, {"dev-shell", {"develop"}}, + {"diff-closures", {"store", "diff-closures"}}, + {"dump-path", {"store", "dump-path"}}, {"hash-file", {"hash", "file"}}, {"hash-path", {"hash", "path"}}, + {"ls-nar", {"nar", "ls"}}, + {"ls-store", {"store", "ls"}}, + {"make-content-addressable", {"store", "make-content-addressable"}}, + {"optimise-store", {"store", "optimise"}}, + {"ping-store", {"store", "ping"}}, + {"sign-paths", {"store", "sign-paths"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, - {"ls-nar", {"nar", "ls"}}, - {"ls-store", {"store", "ls"}}, - {"cat-nar", {"nar", "cat"}}, - {"cat-store", {"store", "cat"}}, - {"dump-path", {"store", "dump-path"}}, + {"verify", {"store", "verify"}}, }; bool aliasUsed = false; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 12c2cf776..0dade90ef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -23,17 +23,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON return { Example{ "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix make-content-addressable nixpkgs#hello" + "nix store make-content-addressable nixpkgs#hello" }, Example{ "To compute a content-addressable representation of the current NixOS system closure:", - "nix make-content-addressable -r /run/current-system" + "nix store make-content-addressable -r /run/current-system" }, }; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); @@ -108,4 +106,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto rCmdMakeContentAddressable = registerCommand("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand2({"store", "make-content-addressable"}); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index 51a7a9756..bc7f175ac 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -18,17 +18,15 @@ struct CmdOptimiseStore : StoreCommand return { Example{ "To optimise the Nix store:", - "nix optimise-store" + "nix store optimise" }, }; } - Category category() override { return catUtility; } - void run(ref store) override { store->optimiseStore(); } }; -static auto rCmdOptimiseStore = registerCommand("optimise-store"); +static auto rCmdOptimiseStore = registerCommand2({"store", "optimise"}); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 8db78d591..19b1a55c8 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -16,17 +16,15 @@ struct CmdPingStore : StoreCommand return { Example{ "To test whether connecting to a remote Nix store via SSH works:", - "nix ping-store --store ssh://mac1" + "nix store ping --store ssh://mac1" }, }; } - Category category() override { return catUtility; } - void run(ref store) override { store->connect(); } }; -static auto rCmdPingStore = registerCommand("ping-store"); +static auto rCmdPingStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 44916c77f..37b8a6712 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,8 +27,6 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -92,7 +90,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto rCmdCopySigs = registerCommand("copy-sigs"); +static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); struct CmdSignPaths : StorePathsCommand { @@ -115,8 +113,6 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } - Category category() override { return catUtility; } - void run(ref store, StorePaths storePaths) override { if (secretKeyFile.empty()) @@ -144,4 +140,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand("sign-paths"); +static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ec7333d03..bcf85d7dd 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -40,17 +40,15 @@ struct CmdVerify : StorePathsCommand return { Example{ "To verify the entire Nix store:", - "nix verify --all" + "nix store verify --all" }, Example{ "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix verify -r -n2 --no-contents $(type -p firefox)" + "nix store verify -r -n2 --no-contents $(type -p firefox)" }, }; } - Category category() override { return catSecondary; } - void run(ref store, StorePaths storePaths) override { std::vector> substituters; @@ -189,4 +187,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto rCmdVerify = registerCommand("verify"); +static auto rCmdVerify = registerCommand2({"store", "verify"}); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 8cb17caf8..92ed36225 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -188,7 +188,7 @@ unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store file://$cacheDir --no-trust # Test local NAR caching. diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 89ff08d4d..7ec25808c 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) +storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 3add896c6..6867f2eb4 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -2,9 +2,9 @@ source common.sh clearStore -garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) -garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) -garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) +garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) +garbage2=$(nix store add-path --name garbage2 ./nar-access.sh) +garbage3=$(nix store add-path --name garbage3 ./nar-access.sh) ls -l $garbage3 POSIXLY_CORRECT=1 du $garbage3 diff --git a/tests/recursive.sh b/tests/recursive.sh index 80a178cc7..b020ec710 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -7,7 +7,7 @@ clearStore rm -f $TEST_ROOT/result -export unreachable=$(nix add-to-store ./recursive.sh) +export unreachable=$(nix store add-path ./recursive.sh) NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' with import ./config.nix; @@ -38,7 +38,7 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r # Add something to the store. echo foobar > foobar - foobar=$(nix $opts add-to-store ./foobar) + foobar=$(nix $opts store add-path ./foobar) nix $opts path-info $foobar nix $opts build $foobar diff --git a/tests/signing.sh b/tests/signing.sh index 9e29e3fbf..bd6280cc6 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -17,40 +17,40 @@ info=$(nix path-info --json $outPath) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] -# Test "nix verify". -nix verify -r $outPath +# Test "nix store verify". +nix store verify -r $outPath -expect 2 nix verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r $outPath --sigs-needed 1 -nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 -expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 -nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" -nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix verify -r $outPath +nix store verify -r $outPath # Verify that the path did not get signed but does have the ultimate bit. info=$(nix path-info --json $outPath2) [[ $info =~ '"ultimate":true' ]] (! [[ $info =~ 'signatures' ]]) -# Test "nix verify". -nix verify -r $outPath2 +# Test "nix store verify". +nix store verify -r $outPath2 -expect 2 nix verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r $outPath2 --sigs-needed 1 -expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix sign-paths". -nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign-paths". +nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 -nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) @@ -59,12 +59,12 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix verify $outPathCA -nix verify $outPathCA --sigs-needed 1000 +nix store verify $outPathCA +nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA -nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. nix copy --to file://$cacheDir $outPath2 @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] @@ -89,17 +89,17 @@ rm -rf $TEST_ROOT/store0 # But succeed if we supply the public keys. nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 -expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath -nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 +nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. (! nix copy --to $TEST_ROOT/store0 $outPath2) nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 # But signatures should still get copied. -nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 # Content-addressed stuff can be copied without signatures. nix copy --to $TEST_ROOT/store0 $outPathCA diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh index dce50974b..053b2f00d 100644 --- a/tests/ssh-relay.sh +++ b/tests/ssh-relay.sh @@ -11,6 +11,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) [ foo = $(< $out) ] From fa8dad10edcebd942ce99e6413fa38e3fe883f15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Dec 2020 23:26:23 +0100 Subject: [PATCH 172/477] Typo --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 75426b2e3..8cf5ccd62 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -413,7 +413,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile return { Example{ "To show what changed between each generation of the NixOS system profile:", - "nix profile diff-closure --profile /nix/var/nix/profiles/system" + "nix profile diff-closures --profile /nix/var/nix/profiles/system" }, }; } From f337aa70998141ccfaa956e9f670152dbb15b385 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Dec 2020 00:58:09 +0100 Subject: [PATCH 173/477] Split 'nix store add-to-store' into 'add-path' and 'add-file' This makes it consistent with 'nix hash '. --- src/nix/{add-to-store.md => add-file.md} | 6 +-- src/nix/add-path.md | 29 +++++++++++ src/nix/add-to-store.cc | 65 ++++++++++++++++-------- tests/fetchurl.sh | 2 +- 4 files changed, 77 insertions(+), 25 deletions(-) rename src/nix/{add-to-store.md => add-file.md} (72%) create mode 100644 src/nix/add-path.md diff --git a/src/nix/add-to-store.md b/src/nix/add-file.md similarity index 72% rename from src/nix/add-to-store.md rename to src/nix/add-file.md index 593ad67ad..ed237a035 100644 --- a/src/nix/add-to-store.md +++ b/src/nix/add-file.md @@ -2,8 +2,8 @@ R""( # Description -Copy the file or directory *path* to the Nix store, and -print the resulting store path on standard output. +Copy the regular file *path* to the Nix store, and print the resulting +store path on standard output. > **Warning** > @@ -18,7 +18,7 @@ Add a regular file to the store: ```console # echo foo > bar -# nix add-to-store --flat ./bar +# nix store add-file ./bar /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar diff --git a/src/nix/add-path.md b/src/nix/add-path.md new file mode 100644 index 000000000..87473611d --- /dev/null +++ b/src/nix/add-path.md @@ -0,0 +1,29 @@ +R""( + +# Description + +Copy *path* to the Nix store, and print the resulting store path on +standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a directory to the store: + +```console +# mkdir dir +# echo foo > dir/bar + +# nix store add-path ./dir +/nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir + +# cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar +foo +``` + +)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 66822b0ff..ea4bbbab9 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,10 +9,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; + FileIngestionMethod ingestionMethod; CmdAddToStore() { + // FIXME: completion expectArg("path", &path); addFlag({ @@ -22,25 +23,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); - - addFlag({ - .longName = "flat", - .shortName = 0, - .description = "add flat file to the Nix store", - .handler = {&ingestionMethod, FileIngestionMethod::Flat}, - }); - } - - std::string description() override - { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-to-store.md" - ; } void run(ref store) override @@ -78,4 +60,45 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto rCmdAddToStore = registerCommand2({"store", "add-path"}); +struct CmdAddFile : CmdAddToStore +{ + CmdAddFile() + { + ingestionMethod = FileIngestionMethod::Flat; + } + + std::string description() override + { + return "add a regular file to the Nix store"; + } + + std::string doc() override + { + return + #include "add-file.md" + ; + } +}; + +struct CmdAddPath : CmdAddToStore +{ + CmdAddPath() + { + ingestionMethod = FileIngestionMethod::Recursive; + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + std::string doc() override + { + return + #include "add-path.md" + ; + } +}; + +static auto rCmdAddFile = registerCommand2({"store", "add-file"}); +static auto rCmdAddPath = registerCommand2({"store", "add-path"}); diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 7ec25808c..10ec0173a 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh) +storePath=$(nix --store $other_store store add-file ./fetchurl.sh) outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) From be09af80026b751569e7dd6519d1564e10e4f34b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 18:03:30 -0600 Subject: [PATCH 174/477] Include static "nix" binary in Hydra build products This allows users to get Nix from Hydra via a stable url like https://hydra.nixos.org/build/132078238/download/1/nix --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 7311a2471..9addccd63 100644 --- a/flake.nix +++ b/flake.nix @@ -466,6 +466,8 @@ postInstall = '' mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products ''; doInstallCheck = true; From 5f66edf24503fabf410bb923de761131cea57771 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 4 Dec 2020 14:28:27 +0100 Subject: [PATCH 175/477] Make `make install` less noisy Remove the printing and useless output of a couple of commands when running `make install` --- doc/manual/local.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index ee5b328de..81a7755e8 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -68,14 +68,14 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/builtins.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp - mv $@.tmp $@ + @mv $@.tmp $@ # Generate the HTML manual. install: $(docdir)/manual/index.html # Generate 'nix' manpages. install: $(d)/src/command-ref/new-cli - for i in doc/manual/src/command-ref/new-cli/*.md; do \ + $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ name=$$(basename $$i .md); \ if [[ $$name = SUMMARY ]]; then continue; fi; \ printf "Title: %s\n\n" "$$name" > $$i.tmp; \ @@ -84,7 +84,7 @@ install: $(d)/src/command-ref/new-cli done $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md - $(trace-gen) mdbook build doc/manual -d $(docdir)/manual + $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual @cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js endif From e20a3ec756c61b076d5fbaea6d976c8ee7fa4c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Kochen?= Date: Fri, 4 Dec 2020 19:32:35 +0100 Subject: [PATCH 176/477] Fix compatibility with newer AWS SDKs Tested against AWS SDK 1.8.99. Fixes #3201. --- configure.ac | 1 + src/libstore/s3-binary-cache-store.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 39306b953..daf378997 100644 --- a/configure.ac +++ b/configure.ac @@ -255,6 +255,7 @@ if test -n "$enable_s3"; then declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) + AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) fi diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4519dd5b5..27253fc12 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -57,6 +57,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { debug("AWS: %s", chomp(statement)); } + +#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) + void Flush() override {} +#endif }; static void initAWS() From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: [PATCH 177/477] Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin") From 692549c542f673015105a3aa4358c1e3095bb0e0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:28:09 -0600 Subject: [PATCH 178/477] Use com.apple.oahd.plist for rosetta 2 detection --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 9c8831d2b..0eaf25bb3 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -47,7 +47,7 @@ case "$(uname -s).$(uname -m)" in ;; Darwin.arm64|Darwin.aarch64) # check for Rosetta 2 support - if ! [ -d /Library/Apple/usr/libexec/oah ]; then + if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" fi From b9a00fd15bb158c293b40aec49c9a426cc4c8921 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 22:17:19 -0600 Subject: [PATCH 179/477] =?UTF-8?q?Canonicalize=20binary=20caches=20with?= =?UTF-8?q?=20=E2=80=98/=E2=80=99=20when=20one=20is=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This checks if there is a trusted substituter with a slash, so trusting https://cache.nixos.org also implies https://cache.nixos.org/ is trusted. --- src/libstore/daemon.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e5cfe94cb..2224d54d5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -215,6 +215,8 @@ struct ClientSettings for (auto & s : ss) if (trusted.count(s)) subs.push_back(s); + else if (!hasSuffix(s, "/") && trusted.count(s + "/")) + subs.push_back(s + "/"); else warn("ignoring untrusted substituter '%s'", s); res = subs; From aa07502009625fe0d38fde1a23c50dd34f1996eb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 6 Dec 2020 23:04:42 -0600 Subject: [PATCH 180/477] Always default to cache.nixos.org even when different nix store dir Since 0744f7f, it is now useful to have cache.nixos.org in substituers even if /nix/store is not the Nix Store Dir. This can always be overridden via configuration, though. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..6b4775683 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -583,7 +583,7 @@ public: Setting substituters{ this, - nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + Strings{"https://cache.nixos.org/"}, "substituters", R"( A list of URLs of substituters, separated by whitespace. The default From 0d7714b0d7f626afc174f1faaf52725290a1247c Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 8 Dec 2020 10:25:03 +0100 Subject: [PATCH 181/477] forgot to add the files --- doc/manual/src/command-ref/nix.md | 1 + doc/manual/src/contributing/contributing.md | 0 2 files changed, 1 insertion(+) create mode 100644 doc/manual/src/command-ref/nix.md create mode 100644 doc/manual/src/contributing/contributing.md diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md new file mode 100644 index 000000000..cc677d127 --- /dev/null +++ b/doc/manual/src/command-ref/nix.md @@ -0,0 +1 @@ +# nix diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md new file mode 100644 index 000000000..e69de29bb From 1b1e0760335832c87516b9103b670b34662d5daf Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 11:11:02 +0100 Subject: [PATCH 182/477] Re-query for the derivation outputs in the post-build-hook We can't assume that the runtime state knows about them as they might have been built remotely, in which case we must query the db again to get them. --- src/libstore/build/derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..fdf777c27 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,8 +899,10 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); + for (auto& [_, maybeOutPath] : + worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (maybeOutPath) + outputPaths.insert(*maybeOutPath); } std::map hookEnvironment = getEnv(); From ae77f21474594fc4555bafa3291424b12ff3d4ac Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 8 Dec 2020 11:59:23 +0100 Subject: [PATCH 183/477] Switch away from classification as Tier1-3 to classification to a more descriptive classification. --- doc/manual/src/contributing/cli-guideline.md | 44 ++++++++++++++------ doc/manual/src/contributing/contributing.md | 1 + doc/manual/src/contributing/hacking.md | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 doc/manual/src/contributing/hacking.md diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 01df07136..0132867c8 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -58,12 +58,13 @@ them. This classification tries to separate commands in 3 categories in terms of their importance in regards to the new users. Users who are likely to be -impacted the most by bad user experience. This does not mean that we will not -take care of Tier3 commands, it means we will only put more helpful details for -beginners into Tier1 commands. +impacted the most by bad user experience. + +- **Main commands** + + Commands used for our main use cases and most likely used by new users. We + expect attention to details, such as: -- **Tier1**: Commands used for our main use cases and most likely used by new - users. From Tier1 commands we expect attention to details, such as: - Proper use of [colors](#colors), [emojis](#special-unicode-characters) and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. @@ -73,16 +74,32 @@ beginners into Tier1 commands. Nix ecosystem. - [Help pages](#help-is-essential) to be as good as we can write them pointing to external documentation and tutorials for more. -- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really - exposing some implementation detail, but not something that we expect a user. - From Tier2 command we expect less attention to details, but still some: + + Examples of such commands: `nix init`, `nix develop`, `nix build`, `nix run`, + ... + +- **Infrequently used commands** + + From infrequently used commands we expect less attention to details, but + still some: + - Proper use of [colors](#colors), [emojis](#special-unicode-characters) and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. -- **Tier3**: Commands that expose certain internal functionality of `nix`, - mostly used by other scripts. + + Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + +- **Utility and scripting commands** + + Commands that expose certain internal functionality of `nix`, mostly used by + other scripts. + - [Autocomplete](#shell-completion) of options. + Examples of such commands: `nix store copy`, `nix hash base16`, `nix store + ping`, ... + + # Help is essential Help should be built into your command line so that new users can gradually @@ -249,9 +266,10 @@ Here are few examples of flag `OPTIONS`: ## Prompt when input not provided -For **Tier1** commands we want command to improve the discoverability of -possible input. A new user will most likely not know which `ARGUMENTS` and -`OPTIONS` are required or which values are possible for those options. +For *main commands* (as [per classification](#classification)) we want command +to improve the discoverability of possible input. A new user will most likely +not know which `ARGUMENTS` and `OPTIONS` are required or which values are +possible for those options. In cases, the user might not provide the input or they provide wrong input, rather then show the error, prompt a user with an option to find and select diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md index e69de29bb..854139a31 100644 --- a/doc/manual/src/contributing/contributing.md +++ b/doc/manual/src/contributing/contributing.md @@ -0,0 +1 @@ +# Contributing diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md new file mode 100644 index 000000000..2ad773dea --- /dev/null +++ b/doc/manual/src/contributing/hacking.md @@ -0,0 +1 @@ +# Hacking From c0f21f08f817745fcf3e9301749b7e237634521c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 06:57:31 +0100 Subject: [PATCH 184/477] Hide the sqlite statements declarations for the local store These have no need to be in the public interface and it causes spurious rebuilds each time one wants to add or remove a new statement. --- src/libstore/local-store.cc | 76 ++++++++++++++++++++++--------------- src/libstore/local-store.hh | 15 +------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 348e5d0d4..2a47b3956 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,6 +42,21 @@ namespace nix { +struct LocalStore::State::Stmts { + /* Some precompiled SQLite statements. */ + SQLiteStmt RegisterValidPath; + SQLiteStmt UpdatePathInfo; + SQLiteStmt AddReference; + SQLiteStmt QueryPathInfo; + SQLiteStmt QueryReferences; + SQLiteStmt QueryReferrers; + SQLiteStmt InvalidatePath; + SQLiteStmt AddDerivationOutput; + SQLiteStmt QueryValidDerivers; + SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryPathFromHashPart; + SQLiteStmt QueryValidPaths; +}; LocalStore::LocalStore(const Params & params) : StoreConfig(params) @@ -60,6 +75,7 @@ LocalStore::LocalStore(const Params & params) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); + state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); @@ -223,31 +239,31 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create(state->db, + state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create(state->db, + state->stmts->UpdatePathInfo.create(state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); - state->stmtAddReference.create(state->db, + state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create(state->db, + state->stmts->QueryPathInfo.create(state->db, "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, + state->stmts->QueryReferences.create(state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - state->stmtQueryReferrers.create(state->db, + state->stmts->QueryReferrers.create(state->db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, + state->stmts->InvalidatePath.create(state->db, "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create(state->db, + state->stmts->AddDerivationOutput.create(state->db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - state->stmtQueryValidDerivers.create(state->db, + state->stmts->QueryValidDerivers.create(state->db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create(state->db, + state->stmts->QueryDerivationOutputs.create(state->db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create(state->db, + state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); + state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); } @@ -590,7 +606,7 @@ void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & out void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { - state.stmtAddDerivationOutput.use() + state.stmts->AddDerivationOutput.use() (deriver) (outputName) (printStorePath(output)) @@ -607,7 +623,7 @@ uint64_t LocalStore::addValidPath(State & state, throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); - state.stmtRegisterValidPath.use() + state.stmts->RegisterValidPath.use() (printStorePath(info.path)) (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) @@ -659,7 +675,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); + auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr(); @@ -679,7 +695,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->registrationTime = useQueryPathInfo.getInt(2); - auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ @@ -687,14 +703,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->ultimate = useQueryPathInfo.getInt(5) == 1; - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); if (s) info->sigs = tokenizeString(s, " "); - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); @@ -709,7 +725,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - state.stmtUpdatePathInfo.use() + state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) @@ -722,7 +738,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); + auto use(state.stmts->QueryPathInfo.use()(printStorePath(path))); if (!use.next()) throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); @@ -731,7 +747,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); + return state.stmts->QueryPathInfo.use()(printStorePath(path)).next(); } @@ -757,7 +773,7 @@ StorePathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); + auto use(state->stmts->QueryValidPaths.use()); StorePathSet res; while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; @@ -767,7 +783,7 @@ StorePathSet LocalStore::queryAllValidPaths() void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); + auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); @@ -788,7 +804,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) return retrySQLite([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); + auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path))); StorePathSet derivers; while (useQueryValidDerivers.next()) @@ -848,7 +864,7 @@ std::map> LocalStore::queryPartialDerivati } auto useQueryDerivationOutputs { - state->stmtQueryDerivationOutputs.use() + state->stmts->QueryDerivationOutputs.use() (drvId) }; @@ -872,11 +888,11 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); + auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix)); if (!useQueryPathFromHashPart.next()) return {}; - const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0); if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) return parseStorePath(s); return {}; @@ -990,7 +1006,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); + state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1030,7 +1046,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(printStorePath(path)).exec(); + state.stmts->InvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 58ec93f27..332718af4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -55,19 +55,8 @@ private: /* The SQLite database object. */ SQLite db; - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - SQLiteStmt stmtQueryValidPaths; + struct Stmts; + std::unique_ptr stmts; /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; From 6758e65612b990805d3d7d2039cd92647730e900 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 09:44:07 +0100 Subject: [PATCH 185/477] Revert "Re-query for the derivation outputs in the post-build-hook" This reverts commit 1b1e0760335832c87516b9103b670b34662d5daf. Using `queryPartialDerivationOutputMap` assumes that the derivation exists locally which isn't the case for remote builders. --- src/libstore/build/derivation-goal.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fdf777c27..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,10 +899,8 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto& [_, maybeOutPath] : - worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (maybeOutPath) - outputPaths.insert(*maybeOutPath); + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); } std::map hookEnvironment = getEnv(); From ee7c94fa1b74b43f7a719373f5f566d09b147126 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 10:37:39 +0100 Subject: [PATCH 186/477] Test the post-build-hook with remote builders Regression test for #4245 --- tests/build-remote-input-addressed.sh | 28 +++++++++++++++++++++++++++ tests/build-remote.sh | 3 +++ 2 files changed, 31 insertions(+) diff --git a/tests/build-remote-input-addressed.sh b/tests/build-remote-input-addressed.sh index b34caa061..49d15c389 100644 --- a/tests/build-remote-input-addressed.sh +++ b/tests/build-remote-input-addressed.sh @@ -3,3 +3,31 @@ source common.sh file=build-hook.nix source build-remote.sh + +# Add a `post-build-hook` option to the nix conf. +# This hook will be executed both for the local machine and the remote builders +# (because they share the same config). +registerBuildHook () { + # Dummy post-build-hook just to ensure that it's executed correctly. + # (we can't reuse the one from `$PWD/push-to-store.sh` because of + # https://github.com/NixOS/nix/issues/4341) + cat < $TEST_ROOT/post-build-hook.sh +#!/bin/sh + +echo "Post hook ran successfully" +# Add an empty line to a counter file, just to check that this hook ran properly +echo "" >> $TEST_ROOT/post-hook-counter +EOF + chmod +x $TEST_ROOT/post-build-hook.sh + rm -f $TEST_ROOT/post-hook-counter + + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf +} + +registerBuildHook +source build-remote.sh + +# `build-hook.nix` has four derivations to build, and the hook runs twice for +# each derivation (once on the builder and once on the host), so the counter +# should contain eight lines now +[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]] diff --git a/tests/build-remote.sh b/tests/build-remote.sh index ca6d1de09..04848e4b5 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -14,6 +14,9 @@ builders=( "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz" ) +chmod -R +w $TEST_ROOT/machine* || true +rm -rf $TEST_ROOT/machine* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). From c87267c2a4621a42d6bfcdb7944cd546f194787a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 10:38:04 +0100 Subject: [PATCH 187/477] Store the final drv outputs in memory when building remotely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `DerivationGoal` has a variable storing the “final” derivation output paths that is used (amongst other things) to fill the environment for the post build hook. However this variable wasn't set when the build-hook is used, causing a crash when both hooks are used together. Fix this by setting this variable (from the informations in the db) after a run of the post build hook. --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..de58d9f06 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2872,6 +2872,8 @@ void DerivationGoal::registerOutputs() for (auto & i : drv->outputsAndOptPaths(worker.store)) { if (!i.second.second || !worker.store.isValidPath(*i.second.second)) allValid = false; + else + finalOutputs.insert_or_assign(i.first, *i.second.second); } if (allValid) return; } From 93a8a005defbe5204f739853403ef11dbc016f33 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 5 Dec 2020 15:33:16 +0100 Subject: [PATCH 188/477] libstore/openStore: fix stores with IPv6 addresses In `nixStable` (2.3.7 to be precise) it's possible to connect to stores using an IPv6 address: nix ping-store --store ssh://root@2001:db8::1 This is also useful for `nixops(1)` where you could specify an IPv6 address in `deployment.targetHost`. However, this behavior is broken on `nixUnstable` and fails with the following error: $ nix store ping --store ssh://root@2001:db8::1 don't know how to open Nix store 'ssh://root@2001:db8::1' This happened because `openStore` from `libstore` uses the `parseURL` function from `libfetchers` which expects a valid URL as defined in RFC2732. However, this is unsupported by `ssh(1)`: $ nix store ping --store 'ssh://root@[2001:db8::1]' cannot connect to 'root@[2001:db8::1]' This patch now allows both ways of specifying a store (`root@2001:db8::1`) and also `root@[2001:db8::1]` since the latter one is useful to pass query parameters to the remote store. In order to achieve this, the following changes were made: * The URL regex from `url-parts.hh` now allows an IPv6 address in the form `2001:db8::1` and also `[2001:db8::1]`. * In `libstore`, a new function named `extractConnStr` ensures that a proper URL is passed to e.g. `ssh(1)`: * If a URL looks like either `[2001:db8::1]` or `root@[2001:db8::1]`, the brackets will be removed using a regex. No additional validation is done here as only strings parsed by `parseURL` are expected. * In any other case, the string will be left untouched. * The rules above only apply for `LegacySSHStore` and `SSHStore` (a.k.a `ssh://` and `ssh-ng://`). Unresolved questions: * I'm not really sure whether we want to allow both variants of IPv6 addresses in the URL parser. However it should be noted that both seem to be possible according to RFC2732: > This document incudes an update to the generic syntax for Uniform > Resource Identifiers defined in RFC 2396 [URL]. It defines a syntax > for IPv6 addresses and allows the use of "[" and "]" within a URI > explicitly for this reserved purpose. * Currently, it's not supported to specify a port number behind the hostname, however it seems as this is not really supported by the URL parser. Hence, this is probably out of scope here. --- src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++++++++- src/libutil/url-parts.hh | 3 ++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 27be66cac..7bf9235b2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,8 @@ #include "archive.hh" #include "callback.hh" +#include + namespace nix { @@ -1091,6 +1093,34 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } } +// The `parseURL` function supports both IPv6 URIs as defined in +// RFC2732, but also pure addresses. The latter one is needed here to +// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). +// +// This function now ensures that a usable connection string is available: +// * If the store to be opened is not an SSH store, nothing will be done. +// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably +// needed to pass further flags), it +// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). +// * If the URL looks like `root@::1` it will be left as-is. +// * In any other case, the string will be left as-is. +static std::string extractConnStr(const std::string &proto, const std::string &connStr) +{ + if (proto.rfind("ssh") != std::string::npos) { + std::smatch result; + std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + + if (std::regex_match(connStr, result, v6AddrRegex)) { + if (result[1].matched) { + return result.str(1) + result.str(3); + } + return result.str(3); + } + } + + return connStr; +} + ref openStore(const std::string & uri_, const Store::Params & extraParams) { @@ -1099,7 +1129,10 @@ ref openStore(const std::string & uri_, auto parsedUri = parseURL(uri_); params.insert(parsedUri.query.begin(), parsedUri.query.end()); - auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + auto baseURI = extractConnStr( + parsedUri.scheme, + parsedUri.authority.value_or("") + parsedUri.path + ); for (auto implem : *Implementations::registered) { if (implem.uriSchemes.count(parsedUri.scheme)) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 68be15cb0..5d21b8d1a 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,8 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+"; +const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; From 5286310e593b2ac13abbbac90c5b458c4be2ad37 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 14:53:45 +0100 Subject: [PATCH 189/477] Use no substituers by default in the tests Otherwise https://cache.nixos.org is chosen by default, causing the OSX testsuite to hang inside the sandbox. (In a way, this is probably rugging an actual bug under the carpet as Nix should be able to gracefully timeout in such a case, but that's beyond mac OSX-fu) --- tests/init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/init.sh b/tests/init.sh index f9ced6b0d..63cf895e2 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -19,6 +19,7 @@ keep-derivations = false sandbox = false experimental-features = nix-command flakes gc-reserved-space = 0 +substituters = flake-registry = $TEST_ROOT/registry.json include nix.conf.extra EOF From a8f533b66417a1025a468cae3068bd2f5c06e811 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Nov 2020 11:19:36 +0100 Subject: [PATCH 190/477] Add lvlNotice log level This is like syslog's LOG_NOTICE: "normal, but significant, condition". --- src/libutil/error.hh | 1 + src/libutil/logging.hh | 1 + src/nix/main.cc | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..aa4fadfcc 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -45,6 +45,7 @@ namespace nix { typedef enum { lvlError = 0, lvlWarn, + lvlNotice, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 82ba54051..96ad69790 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -198,6 +198,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ } while (0) #define printError(args...) printMsg(lvlError, args) +#define notice(args...) printMsg(lvlNotice, args) #define printInfo(args...) printMsg(lvlInfo, args) #define printTalkative(args...) printMsg(lvlTalkative, args) #define debug(args...) printMsg(lvlDebug, args) diff --git a/src/nix/main.cc b/src/nix/main.cc index fb3bffeaf..27b1d7257 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -250,7 +250,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlWarn; + verbosity = lvlNotice; settings.verboseBuild = false; evalSettings.pureEval = true; From c6a1bcd0ec1ed443947ae7151e32dd6827dfe53e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 17:11:39 +0100 Subject: [PATCH 191/477] nix store make-content-addressable: Show rewritten path --- src/nix/make-content-addressable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0dade90ef..5165c4804 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }; if (!json) - printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); From eb453081092cbee5f8176c1d348ac23e46a24281 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 10 Dec 2020 17:40:00 +0100 Subject: [PATCH 192/477] Fix the `nix` command with CA derivations Prevents a crash because most `nix` subcommands assumed that derivations know their output path, which isn't the case for CA derivations --- src/nix/installables.cc | 2 +- tests/content-addressed.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b6ed030af..3506c3fcc 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -409,7 +409,7 @@ std::vector InstallableAttrPath::toDerivations for (auto & drvInfo : drvInfos) { res.push_back({ state->store->parseStorePath(drvInfo.queryDrvPath()), - state->store->parseStorePath(drvInfo.queryOutPath()), + state->store->maybeParseStorePath(drvInfo.queryOutPath()), drvInfo.queryOutputName() }); } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 03eff549c..bc37a99c1 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -50,7 +50,13 @@ testGC () { nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true } +testNixCommand () { + clearStore + nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link +} + testRemoteCache testDeterministicCA testCutoff testGC +testNixCommand From 63b3536f50f124cdcd7592b344eac157a1439d42 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Fri, 28 Aug 2020 10:28:35 -0300 Subject: [PATCH 193/477] treat s3 permission errors as file-not-found Signed-off-by: Jonathan Ringer --- src/libstore/s3-binary-cache-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27253fc12..d6edafd7e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject( dynamic_cast(result.GetBody()).str()); } catch (S3Error & e) { - if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) && + (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw; } auto now2 = std::chrono::steady_clock::now(); From 58cdab64acd4807f73768fb32acdde39b501799f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Oct 2020 17:36:51 +0200 Subject: [PATCH 194/477] Store metadata about drv outputs realisations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each known realisation, store: - its output - its output path This comes with a set of needed changes: - New `realisations` module declaring the types needed for describing these mappings - New `Store::registerDrvOutput` method registering all the needed informations about a derivation output (also replaces `LocalStore::linkDeriverToPath`) - new `Store::queryRealisation` method to retrieve the informations for a derivations This introcudes some redundancy on the remote-store side between `wopQueryDerivationOutputMap` and `wopQueryRealisation`. However we might need to keep both (regardless of backwards compat) because we sometimes need to get some infos for all the outputs of a derivation (where `wopQueryDerivationOutputMap` is handy), but all the stores can't implement it − because listing all the outputs of a derivation isn't really possible for binary caches where the server doesn't allow to list a directory. --- src/libstore/binary-cache-store.cc | 17 ++++++ src/libstore/binary-cache-store.hh | 7 +++ src/libstore/build/derivation-goal.cc | 19 ++++++- src/libstore/daemon.cc | 22 ++++++++ src/libstore/dummy-store.cc | 3 + src/libstore/legacy-ssh-store.cc | 4 ++ src/libstore/local-binary-cache-store.cc | 1 + src/libstore/local-store.cc | 21 +++++-- src/libstore/local-store.hh | 12 ++-- src/libstore/realisation.cc | 72 ++++++++++++++++++++++++ src/libstore/realisation.hh | 34 +++++++++++ src/libstore/remote-store.cc | 21 +++++++ src/libstore/remote-store.hh | 4 ++ src/libstore/store-api.hh | 15 +++++ src/libstore/worker-protocol.hh | 5 ++ tests/content-addressed.sh | 3 +- 16 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 src/libstore/realisation.cc create mode 100644 src/libstore/realisation.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a918b7208..085dc7ba1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -443,6 +443,23 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } +std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) +{ + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; + auto rawOutputInfo = getFile(outputInfoFilePath); + + if (rawOutputInfo) { + return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + } else { + return std::nullopt; + } +} + +void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; + upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); +} + ref BinaryCacheStore::getFSAccessor() { return make_ref(ref(shared_from_this()), localNarCache); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5224d7ec8..07a8b2beb 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -33,6 +33,9 @@ private: protected: + // The prefix under which realisation infos will be stored + const std::string realisationsPrefix = "/realisations"; + BinaryCacheStore(const Params & params); public: @@ -99,6 +102,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void narFromPath(const StorePath & path, Sink & sink) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de58d9f06..a0f10c33d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2094,6 +2094,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf /* Nothing to be done; 'path' must already be valid. */ } + void registerDrvOutput(const Realisation & info) override + { + if (!goal.isAllowed(info.id.drvPath)) + throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + next->registerDrvOutput(info); + } + + std::optional queryRealisation(const DrvOutput & id) override + { + if (!goal.isAllowed(id.drvPath)) + throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + return next->queryRealisation(id); + } + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -3393,7 +3407,10 @@ void DerivationGoal::registerOutputs() if (useDerivation || isCaFloating) for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); + worker.store.registerDrvOutput( + DrvOutputId{drvPathResolved, outputName}, + DrvOutputInfo{.outPath = newInfo.path, + .resolvedDrv = drvPathResolved}); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2224d54d5..ba5788b64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopRegisterDrvOutput: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + auto resolvedDrv = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + logger->stopWork(); + break; + } + + case wopQueryRealisation: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto info = store->queryRealisation(outputId); + logger->stopWork(); + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + break; + } + default: throw Error("invalid operation %1%", op); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 98b745c3a..91fc178db 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } + + std::optional queryRealisation(const DrvOutput&) override + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 467169ce8..ad1779aea 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -333,6 +333,10 @@ public: auto conn(connections->get()); return conn->remoteVersion; } + + std::optional queryRealisation(const DrvOutput&) override + // TODO: Implement + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regLegacySSHStore; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 7d979c5c2..bb7464989 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -87,6 +87,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + createDirs(binaryCacheDir + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a47b3956..418b3ab9c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -597,13 +597,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } -void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); + // XXX: This ignores the references of the output because we can + // recompute them later from the drv and the references of the associated + // store path, but doing so is both inefficient and fragile. + return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); } -void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -653,7 +656,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - linkDeriverToPath(state, id, i.first, *i.second.second); + registerDrvOutput_(state, id, i.first, *i.second.second); } } @@ -1612,5 +1615,13 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } - +std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { + auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); + if (!(outputPath && isValidPath(*outputPath))) + return std::nullopt; + else + return {DrvOutputInfo{ + .outPath = *outputPath, + }}; } +} // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 332718af4..440411f01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -208,6 +208,13 @@ public: garbage until it exceeds maxFree. */ void autoGC(bool sync = true); + /* Register the store path 'output' as the output named 'outputName' of + derivation 'deriver'. */ + void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; + void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + + std::optional queryRealisation(const DrvOutput&) override; + private: int getSchema(); @@ -276,11 +283,6 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ - void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output); - void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output); - Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc new file mode 100644 index 000000000..fcc1a3825 --- /dev/null +++ b/src/libstore/realisation.cc @@ -0,0 +1,72 @@ +#include "realisation.hh" +#include "store-api.hh" + +namespace nix { + +MakeError(InvalidDerivationOutputId, Error); + +DrvOutput DrvOutput::parse(const std::string &strRep) { + const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); + if (outputs.size() != 1) + throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); + + return DrvOutput{ + .drvPath = StorePath(rawPath), + .outputName = *outputs.begin(), + }; +} + +std::string DrvOutput::to_string() const { + return std::string(drvPath.to_string()) + "!" + outputName; +} + +std::string Realisation::to_string() const { + std::string res; + + res += "Id: " + id.to_string() + '\n'; + res += "OutPath: " + std::string(outPath.to_string()) + '\n'; + + return res; +} + +Realisation Realisation::parse(const std::string & s, const std::string & whence) +{ + // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out + auto corrupt = [&]() { + return Error("Drv output info file '%1%' is corrupt", whence); + }; + + std::optional id; + std::optional outPath; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) throw corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) throw corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "Id") + id = DrvOutput::parse(value); + + if (name == "OutPath") + outPath = StorePath(value); + + pos = eol + 1; + } + + if (!outPath) corrupt(); + if (!id) corrupt(); + return Realisation { + .id = *id, + .outPath = *outPath, + }; +} + +} // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh new file mode 100644 index 000000000..c573e1bb4 --- /dev/null +++ b/src/libstore/realisation.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct DrvOutput { + StorePath drvPath; + std::string outputName; + + std::string to_string() const; + + static DrvOutput parse(const std::string &); + + bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } + bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } + +private: + // Just to make comparison operators easier to write + std::pair to_pair() const + { return std::make_pair(drvPath, outputName); } +}; + +struct Realisation { + DrvOutput id; + StorePath outPath; + + std::string to_string() const; + static Realisation parse(const std::string & s, const std::string & whence); +}; + +typedef std::map DrvOutputs; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be29f8e6f..f1f4d0516 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } +void RemoteStore::registerDrvOutput(const Realisation & info) +{ + auto conn(getConnection()); + conn->to << wopRegisterDrvOutput; + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + conn.processStderr(); +} + +std::optional RemoteStore::queryRealisation(const DrvOutput & id) +{ + auto conn(getConnection()); + conn->to << wopQueryRealisation; + conn->to << id.to_string(); + conn.processStderr(); + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; +} + void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f78fcb02..fdd53e6ed 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -81,6 +81,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6b9331495..7cdadc1f3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "realisation.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -396,6 +397,8 @@ protected: public: + virtual std::optional queryRealisation(const DrvOutput &) = 0; + /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -468,6 +471,18 @@ public: virtual StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) = 0; + /** + * Add a mapping indicating that `deriver!outputName` maps to the output path + * `output`. + * + * This is redundant for known-input-addressed and fixed-output derivations + * as this information is already present in the drv file, but necessary for + * floating-ca derivations and their dependencies as there's no way to + * retrieve this information otherwise. + */ + virtual void registerDrvOutput(const Realisation & output) + { unsupported("registerDrvOutput"); } + /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 63bd6ea49..f2cdc7ca3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,5 +1,8 @@ #pragma once +#include "store-api.hh" +#include "serialise.hh" + namespace nix { @@ -50,6 +53,8 @@ typedef enum { wopAddToStoreNar = 39, wopQueryMissing = 40, wopQueryDerivationOutputMap = 41, + wopRegisterDrvOutput = 42, + wopQueryRealisation = 43, } WorkerOp; diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index bc37a99c1..e8ac88609 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -55,7 +55,8 @@ testNixCommand () { nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link } -testRemoteCache +# Disabled until we have it properly working +# testRemoteCache testDeterministicCA testCutoff testGC From 3ac9d74eb1de0f696bb0384132f7ecc7d057f9d6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:14:02 +0200 Subject: [PATCH 195/477] Rework the db schema for derivation outputs Add a new table for tracking the derivation output mappings. We used to hijack the `DerivationOutputs` table for that, but (despite its name), it isn't a really good fit: - Its entries depend on the drv being a valid path, making it play badly with garbage collection and preventing us to copy a drv output without copying the whole drv closure too; - It dosen't guaranty that the output path exists; By using a different table, we can experiment with a different schema better suited for tracking the output mappings of CA derivations. (incidentally, this also fixes #4138) --- src/libstore/build/derivation-goal.cc | 16 +- src/libstore/ca-specific-schema.sql | 11 ++ src/libstore/local-store.cc | 217 ++++++++++++++++++-------- src/libstore/local-store.hh | 4 +- src/libstore/local.mk | 4 +- 5 files changed, 172 insertions(+), 80 deletions(-) create mode 100644 src/libstore/ca-specific-schema.sql diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a0f10c33d..b7bf866eb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && + ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed)) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3405,12 +3406,11 @@ void DerivationGoal::registerOutputs() drvPathResolved = writeDerivation(worker.store, drv2); } - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.registerDrvOutput( - DrvOutputId{drvPathResolved, outputName}, - DrvOutputInfo{.outPath = newInfo.path, - .resolvedDrv = drvPathResolved}); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + for (auto& [outputName, newInfo] : infos) + worker.store.registerDrvOutput(Realisation{ + .id = DrvOutput{drvPathResolved, outputName}, + .outPath = newInfo.path}); } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql new file mode 100644 index 000000000..93c442826 --- /dev/null +++ b/src/libstore/ca-specific-schema.sql @@ -0,0 +1,11 @@ +-- Extension of the sql schema for content-addressed derivations. +-- Won't be loaded unless the experimental feature `ca-derivations` +-- is enabled + +create table if not exists Realisations ( + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + primary key (drvPath, outputName), + foreign key (outputPath) references ValidPaths(id) on delete cascade +); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 418b3ab9c..69ab821d9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -52,12 +52,52 @@ struct LocalStore::State::Stmts { SQLiteStmt QueryReferrers; SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; + SQLiteStmt RegisterRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryRealisedOutput; + SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; }; +int getSchema(Path schemaPath) +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error("'%1%' is corrupt", schemaPath); + } + return curSchema; +} + +void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) +{ + const int nixCASchemaVersion = 1; + int curCASchema = getSchema(schemaPath); + if (curCASchema != nixCASchemaVersion) { + if (curCASchema > nixCASchemaVersion) { + throw Error("current Nix store ca-schema is version %1%, but I only support %2%", + curCASchema, nixCASchemaVersion); + } + + if (!lockFile(lockFd.get(), ltWrite, false)) { + printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltWrite, true); + } + + if (curCASchema == 0) { + static const char schema[] = + #include "ca-specific-schema.sql.gen.hh" + ; + db.exec(schema); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + lockFile(lockFd.get(), ltRead, true); + } +} + LocalStore::LocalStore(const Params & params) : StoreConfig(params) , Store(params) @@ -238,6 +278,10 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } + /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); @@ -264,6 +308,28 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + state->stmts->RegisterRealisedOutput.create(state->db, + R"( + insert or replace into Realisations (drvPath, outputName, outputPath) + values (?, ?, (select id from ValidPaths where path = ?)) + ; + )"); + state->stmts->QueryRealisedOutput.create(state->db, + R"( + select Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? and outputName = ? + ; + )"); + state->stmts->QueryAllRealisedOutputs.create(state->db, + R"( + select outputName, Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? + ; + )"); + } } @@ -301,16 +367,7 @@ std::string LocalStore::getUri() int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error("'%1%' is corrupt", schemaPath); - } - return curSchema; -} - +{ return nix::getSchema(schemaPath); } void LocalStore::openDB(State & state, bool create) { @@ -600,13 +657,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - // XXX: This ignores the references of the output because we can - // recompute them later from the drv and the references of the associated - // store path, but doing so is both inefficient and fragile. - return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); + retrySQLite([&]() { + state->stmts->RegisterRealisedOutput.use() + (info.id.drvPath.to_string()) + (info.id.outputName) + (printStorePath(info.outPath)) + .exec(); + }); } -void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -656,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - registerDrvOutput_(state, id, i.first, *i.second.second); + cacheDrvOutputMapping(state, id, i.first, *i.second.second); } } @@ -817,70 +877,85 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } - -std::map> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +// Try to resolve the derivation at path `original`, with a caching layer +// to make it more efficient +std::optional cachedResolve( + LocalStore & store, + const StorePath & original) { - auto path = path_; - std::map> outputs; - Derivation drv = readDerivation(path); - for (auto & [outName, _] : drv.outputs) { - outputs.insert_or_assign(outName, std::nullopt); - } - bool haveCached = false; { auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(path); + auto resolvedPathOptIter = resolutions->find(original); if (resolvedPathOptIter != resolutions->end()) { auto & [_, resolvedPathOpt] = *resolvedPathOptIter; if (resolvedPathOpt) - path = *resolvedPathOpt; - haveCached = true; + return resolvedPathOpt; } } - /* can't just use else-if instead of `!haveCached` because we need to unlock - `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { - /* Try resolve drv and use that path instead. */ - auto attempt = drv.tryResolve(*this); - if (!attempt) - /* If we cannot resolve the derivation, we cannot have any path - assigned so we return the map of all std::nullopts. */ - return outputs; - /* Just compute store path */ - auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); - /* Store in memo table. */ - /* FIXME: memo logic should not be local-store specific, should have - wrapper-method instead. */ - drvPathResolutions.lock()->insert_or_assign(path, pathResolved); - path = std::move(pathResolved); - } - return retrySQLite>>([&]() { - auto state(_state.lock()); + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(original); + auto attempt = drv.tryResolve(store); + if (!attempt) + return std::nullopt; + /* Just compute store path */ + auto pathResolved = + writeDerivation(store, *std::move(attempt), NoRepair, true); + /* Store in memo table. */ + drvPathResolutions.lock()->insert_or_assign(original, pathResolved); + return pathResolved; +} + +std::map> +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +{ + auto path = path_; + auto outputs = retrySQLite>>([&]() { + auto state(_state.lock()); + std::map> outputs; uint64_t drvId; try { drvId = queryValidPathId(*state, path); - } catch (InvalidPath &) { - /* FIXME? if the derivation doesn't exist, we cannot have a mapping - for it. */ - return outputs; + } catch (InvalidPath&) { + // Ignore non-existing drvs as they might still have an output map + // defined if ca-derivations is enabled } + auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); + while (use.next()) + outputs.insert_or_assign( + use.getStr(0), parseStorePath(use.getStr(1))); - auto useQueryDerivationOutputs { - state->stmts->QueryDerivationOutputs.use() - (drvId) - }; + return outputs; + }); + + if (!settings.isExperimentalFeatureEnabled("ca-derivations")) + return outputs; + + auto drv = readDerivation(path); + + for (auto & output : drv.outputsAndOptPaths(*this)) { + outputs.emplace(output.first, std::nullopt); + } + + auto resolvedDrv = cachedResolve(*this, path); + + if (!resolvedDrv) + return outputs; + + retrySQLite([&]() { + auto state(_state.lock()); + path = *resolvedDrv; + auto useQueryDerivationOutputs{ + state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; while (useQueryDerivationOutputs.next()) outputs.insert_or_assign( useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1)) - ); - - return outputs; + parseStorePath(useQueryDerivationOutputs.getStr(1))); }); -} + return outputs; +} std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { @@ -1615,13 +1690,19 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { - auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); - if (!(outputPath && isValidPath(*outputPath))) - return std::nullopt; - else - return {DrvOutputInfo{ - .outPath = *outputPath, - }}; +std::optional LocalStore::queryRealisation( + const DrvOutput& id) { + typedef std::optional Ret; + return retrySQLite([&]() -> Ret { + auto state(_state.lock()); + auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + id.outputName)); + if (!use.next()) + return std::nullopt; + auto outputPath = parseStorePath(use.getStr(0)); + auto resolvedDrv = StorePath(use.getStr(1)); + return Ret{ + Realisation{.id = id, .outPath = outputPath}}; + }); } } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 440411f01..69559e346 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -210,8 +210,8 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ - void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; - void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + void registerDrvOutput(const Realisation & info) override; + void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); std::optional queryRealisation(const DrvOutput&) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index dfe1e2cc4..03c4351ac 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif -$(d)/local-store.cc: $(d)/schema.sql.gen.hh +$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(d)/build.cc: @@ -58,7 +58,7 @@ $(d)/build.cc: @echo ')foo"' >> $@.tmp @mv $@.tmp $@ -clean-files += $(d)/schema.sql.gen.hh +clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) From 8914e01e37ad072d940e2000fede7c2e0f4b194c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 21:07:52 +0100 Subject: [PATCH 196/477] Store the realisations as JSON in the binary cache Fix #4332 --- src/libstore/binary-cache-store.cc | 5 ++- src/libstore/realisation.cc | 61 ++++++++++-------------------- src/libstore/realisation.hh | 5 ++- 3 files changed, 25 insertions(+), 46 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 085dc7ba1..5b081c1ae 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -449,7 +449,8 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + return {Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; } else { return std::nullopt; } @@ -457,7 +458,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); + upsertFile(filePath, info.toJSON(), "application/json"); } ref BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index fcc1a3825..47db1ec9f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include namespace nix { @@ -20,52 +21,28 @@ std::string DrvOutput::to_string() const { return std::string(drvPath.to_string()) + "!" + outputName; } -std::string Realisation::to_string() const { - std::string res; - - res += "Id: " + id.to_string() + '\n'; - res += "OutPath: " + std::string(outPath.to_string()) + '\n'; - - return res; +nlohmann::json Realisation::toJSON() const { + return nlohmann::json{ + {"id", id.to_string()}, + {"outPath", outPath.to_string()}, + }; } -Realisation Realisation::parse(const std::string & s, const std::string & whence) -{ - // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out - auto corrupt = [&]() { - return Error("Drv output info file '%1%' is corrupt", whence); +Realisation Realisation::fromJSON( + const nlohmann::json& json, + const std::string& whence) { + auto getField = [&](std::string fieldName) -> std::string { + auto fieldIterator = json.find(fieldName); + if (fieldIterator == json.end()) + throw Error( + "Drv output info file '%1%' is corrupt, missing field %2%", + whence, fieldName); + return *fieldIterator; }; - std::optional id; - std::optional outPath; - - size_t pos = 0; - while (pos < s.size()) { - - size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); - - std::string name(s, pos, colon - pos); - - size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); - - std::string value(s, colon + 2, eol - colon - 2); - - if (name == "Id") - id = DrvOutput::parse(value); - - if (name == "OutPath") - outPath = StorePath(value); - - pos = eol + 1; - } - - if (!outPath) corrupt(); - if (!id) corrupt(); - return Realisation { - .id = *id, - .outPath = *outPath, + return Realisation{ + .id = DrvOutput::parse(getField("id")), + .outPath = StorePath(getField("outPath")), }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index c573e1bb4..08579b739 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,6 +1,7 @@ #pragma once #include "path.hh" +#include namespace nix { @@ -25,8 +26,8 @@ struct Realisation { DrvOutput id; StorePath outPath; - std::string to_string() const; - static Realisation parse(const std::string & s, const std::string & whence); + nlohmann::json toJSON() const; + static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); }; typedef std::map DrvOutputs; From bab1cda0e6c30e25460b5a9c809589d3948f35df Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 16:56:56 +0100 Subject: [PATCH 197/477] Use the hash modulo in the derivation outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than storing the derivation outputs as `drvPath!outputName` internally, store them as `drvHashModulo!outputName` (or `outputHash!outputName` for fixed-output derivations). This makes the storage slightly more opaque, but enables an earlier cutoff in cases where a fixed-output dependency changes (but keeps the same output hash) − same as what we already do for input-addressed derivations. --- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-goal.cc | 28 ++++-------- src/libstore/derivations.cc | 49 +++++++++++++++------ src/libstore/derivations.hh | 22 ++++------ src/libstore/local-store.cc | 61 ++++++++++++--------------- src/libstore/realisation.cc | 10 ++--- src/libstore/realisation.hh | 10 +++-- 7 files changed, 93 insertions(+), 89 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..d059e3daf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1107,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // Shouldn't happen as the toplevel derivation is not CA. assert(false); }, - [&](UnknownHashes) { + [&](DeferredHash _) { for (auto & i : outputs) { drv.outputs.insert_or_assign(i, DerivationOutput { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b7bf866eb..54b37553a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -504,9 +504,6 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -2097,15 +2094,15 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf void registerDrvOutput(const Realisation & info) override { - if (!goal.isAllowed(info.id.drvPath)) - throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how next->registerDrvOutput(info); } std::optional queryRealisation(const DrvOutput & id) override { - if (!goal.isAllowed(id.drvPath)) - throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how return next->queryRealisation(id); } @@ -3394,23 +3391,14 @@ void DerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (settings.isExperimentalFeatureEnabled("ca-derivations")) + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto outputHashes = staticOutputHashes(worker.store, *drv); for (auto& [outputName, newInfo] : infos) worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{drvPathResolved, outputName}, + .id = DrvOutput{outputHashes.at(outputName), outputName}, .outPath = newInfo.path}); + } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 231ca26c2..5bcc7f012 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { + bool isDeferred = false; /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { - case DerivationType::CAFloating: - return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } + case DerivationType::CAFloating: + isDeferred = true; + break; case DerivationType::InputAddressed: break; case DerivationType::DeferredInputAddressed: @@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { - bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation [&](Hash drvHash) { inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); }, + [&](DeferredHash deferredHash) { + isDeferred = true; + inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); + }, // CA derivation's output hashes [&](CaOutputHashes outputHashes) { std::set justOut = { "out" }; @@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, - [&](UnknownHashes) { - hasUnknownHash = true; - }, }, res); - if (hasUnknownHash) { - return UnknownHashes {}; - } } - return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + + if (isDeferred) + return DeferredHash { hash }; + else + return hash; +} + + +std::map staticOutputHashes(Store& store, const Derivation& drv) +{ + std::map res; + std::visit(overloaded { + [&](Hash drvHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, drvHash}); + } + }, + [&](DeferredHash deferredHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, deferredHash.hash}); + } + }, + [&](CaOutputHashes outputHashes) { + res = outputHashes; + }, + }, hashDerivationModulo(store, drv, true)); + return res; } @@ -719,9 +745,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } - -Sync drvPathResolutions; - std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b966d6d90..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -18,8 +18,6 @@ namespace nix { /* The traditional non-fixed-output derivation type. */ struct DerivationOutputInputAddressed { - /* Will need to become `std::optional` once input-addressed - derivations are allowed to depend on cont-addressed derivations */ StorePath path; }; @@ -174,12 +172,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; -struct UnknownHashes {}; +struct DeferredHash { Hash hash; }; typedef std::variant< Hash, // regular DRV normalized hash CaOutputHashes, // Fixed-output derivation hashes - UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies + DeferredHash // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations @@ -207,22 +205,18 @@ typedef std::variant< */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +/* + Return a map associating each output to a hash that uniquely identifies its + derivation (modulo the self-references). + */ +std::map staticOutputHashes(Store& store, const Derivation& drv); + /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; -/* Memoisation of `readDerivation(..).resove()`. */ -typedef std::map< - StorePath, - std::optional -> DrvPathResolutions; - -// FIXME: global, though at least thread-safe. -// FIXME: arguably overlaps with hashDerivationModulo memo table. -extern Sync drvPathResolutions; - bool wantOutput(const string & output, const std::set & wanted); struct Source; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 69ab821d9..1539c94e2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -659,7 +659,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() - (info.id.drvPath.to_string()) + (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) .exec(); @@ -879,17 +879,18 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) // Try to resolve the derivation at path `original`, with a caching layer // to make it more efficient -std::optional cachedResolve( - LocalStore & store, - const StorePath & original) +std::optional cachedResolve( + LocalStore& store, + const StorePath& original) { + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; { - auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(original); - if (resolvedPathOptIter != resolutions->end()) { - auto & [_, resolvedPathOpt] = *resolvedPathOptIter; - if (resolvedPathOpt) - return resolvedPathOpt; + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(original); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; } } @@ -898,12 +899,9 @@ std::optional cachedResolve( auto attempt = drv.tryResolve(store); if (!attempt) return std::nullopt; - /* Just compute store path */ - auto pathResolved = - writeDerivation(store, *std::move(attempt), NoRepair, true); /* Store in memo table. */ - drvPathResolutions.lock()->insert_or_assign(original, pathResolved); - return pathResolved; + resolutionsCache.lock()->insert_or_assign(original, *attempt); + return *attempt; } std::map> @@ -933,26 +931,24 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto drv = readDerivation(path); - for (auto & output : drv.outputsAndOptPaths(*this)) { - outputs.emplace(output.first, std::nullopt); - } - auto resolvedDrv = cachedResolve(*this, path); - if (!resolvedDrv) + if (!resolvedDrv) { + for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { + if (!outputs.count(outputName)) + outputs.emplace(outputName, std::nullopt); + } return outputs; + } - retrySQLite([&]() { - auto state(_state.lock()); - path = *resolvedDrv; - auto useQueryDerivationOutputs{ - state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; - - while (useQueryDerivationOutputs.next()) - outputs.insert_or_assign( - useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1))); - }); + auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); + for (auto& [outputName, hash] : resolvedDrvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) + outputs.insert_or_assign(outputName, realisation->outPath); + else + outputs.insert_or_assign(outputName, std::nullopt); + } return outputs; } @@ -1695,12 +1691,11 @@ std::optional LocalStore::queryRealisation( typedef std::optional Ret; return retrySQLite([&]() -> Ret { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( id.outputName)); if (!use.next()) return std::nullopt; auto outputPath = parseStorePath(use.getStr(0)); - auto resolvedDrv = StorePath(use.getStr(1)); return Ret{ Realisation{.id = id, .outPath = outputPath}}; }); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47db1ec9f..47ad90eee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -7,18 +7,18 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); DrvOutput DrvOutput::parse(const std::string &strRep) { - const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); - if (outputs.size() != 1) + size_t n = strRep.find("!"); + if (n == strRep.npos) throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); return DrvOutput{ - .drvPath = StorePath(rawPath), - .outputName = *outputs.begin(), + .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), + .outputName = strRep.substr(n+1), }; } std::string DrvOutput::to_string() const { - return std::string(drvPath.to_string()) + "!" + outputName; + return strHash() + "!" + outputName; } nlohmann::json Realisation::toJSON() const { diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 08579b739..4b8ead3c5 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -6,11 +6,15 @@ namespace nix { struct DrvOutput { - StorePath drvPath; + // The hash modulo of the derivation + Hash drvHash; std::string outputName; std::string to_string() const; + std::string strHash() const + { return drvHash.to_string(Base16, true); } + static DrvOutput parse(const std::string &); bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } @@ -18,8 +22,8 @@ struct DrvOutput { private: // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvPath, outputName); } + std::pair to_pair() const + { return std::make_pair(drvHash, outputName); } }; struct Realisation { From e9b39f6004ec68f062230514534b08033cf133c7 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Dec 2020 21:12:53 +0100 Subject: [PATCH 198/477] Restrict the operations on drv outputs in recursive Nix There's currently no way to properly filter them, so disallow them altogether instead. --- src/libstore/build/derivation-goal.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 54b37553a..f494545fb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2093,18 +2093,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf } void registerDrvOutput(const Realisation & info) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - next->registerDrvOutput(info); - } + // XXX: This should probably be allowed as a no-op if the realisation + // corresponds to an allowed derivation + { throw Error("registerDrvOutput"); } std::optional queryRealisation(const DrvOutput & id) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - return next->queryRealisation(id); - } + // XXX: This should probably be allowed if the realisation corresponds to + // an allowed derivation + { throw Error("queryRealisation"); } void buildPaths(const std::vector & paths, BuildMode buildMode) override { From fa307875e961a616a049206645a651a76a050a79 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 11 Dec 2020 23:32:45 +0100 Subject: [PATCH 199/477] Introduce NormalType for the normal type of a Value This will be useful to abstract over the ValueType implementation details Make use of it already to replace the showType(ValueType) function --- src/libexpr/eval-cache.cc | 4 ++-- src/libexpr/eval.cc | 34 ++++++++++++++++------------------ src/libexpr/eval.hh | 2 +- src/libexpr/flake/flake.cc | 26 +++++++++++++------------- src/libexpr/value.hh | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 34 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..a11327f77 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -513,7 +513,7 @@ std::string AttrCursor::getString() auto & v = forceValue(); if (v.type != tString && v.type != tPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); return v.type == tString ? v.string.s : v.path; } @@ -548,7 +548,7 @@ string_t AttrCursor::getStringWithContext() else if (v.type == tPath) return {v.path, {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); } bool AttrCursor::getBool() diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c6f4d1716..48fe0bbda 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -165,25 +165,20 @@ const Value *getPrimOp(const Value &v) { return primOp; } - -string showType(ValueType type) +string showType(NormalType type) { switch (type) { - case tInt: return "an integer"; - case tBool: return "a Boolean"; - case tString: return "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList1: case tList2: case tListN: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; - case tExternal: return "an external value"; - case tFloat: return "a float"; + case nInt: return "an integer"; + case nBool: return "a Boolean"; + case nString: return "a string"; + case nPath: return "a path"; + case nNull: return "null"; + case nAttrs: return "a set"; + case nList: return "a list"; + case nFunction: return "a function"; + case nExternal: return "an external value"; + case nFloat: return "a float"; + case nThunk: return "a thunk"; } abort(); } @@ -198,8 +193,11 @@ string showType(const Value & v) case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tBlackhole: return "a black hole"; default: - return showType(v.type); + return showType(v.normalType()); } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e1f61baa..211529954 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -346,7 +346,7 @@ private: /* Return a string representing the type of the value `v'. */ -string showType(ValueType type); +string showType(NormalType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair parseFlakeInputs( @@ -93,7 +93,7 @@ static std::map parseFlakeInputs( static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const Pos & pos) { - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); FlakeInput input; @@ -108,16 +108,16 @@ static FlakeInput parseFlakeInput(EvalState & state, for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); url = attr.value->string.s; attrs.emplace("url", *url); } else if (attr.name == sFlake) { - expectType(state, tBool, *attr.value, *attr.pos); + expectType(state, nBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); } else if (attr.name == sFollows) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { if (attr.value->type == tString) @@ -158,7 +158,7 @@ static std::map parseFlakeInputs( { std::map inputs; - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); for (nix::Attr & inputAttr : *(*value).attrs) { inputs.emplace(inputAttr.name, @@ -199,10 +199,10 @@ static Flake getFlake( Value vInfo; state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack - expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); if (auto description = vInfo.attrs->get(state.sDescription)) { - expectType(state, tString, *description->value, *description->pos); + expectType(state, nString, *description->value, *description->pos); flake.description = description->value->string.s; } @@ -214,9 +214,9 @@ static Flake getFlake( auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { - expectType(state, tLambda, *outputs->value, *outputs->pos); + expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { @@ -231,7 +231,7 @@ static Flake getFlake( auto sNixConfig = state.symbols.create("nixConfig"); if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { - expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); + expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos); for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index fe11bb2ed..833af0f3d 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -29,6 +29,22 @@ typedef enum { tFloat } ValueType; +// This type abstracts over all actual value types in the language, +// grouping together implementation details like tList*, different function +// types, and types in non-normal form (so thunks and co.) +typedef enum { + nThunk, + nInt, + nFloat, + nBool, + nString, + nPath, + nNull, + nAttrs, + nList, + nFunction, + nExternal +} NormalType; class Bindings; struct Env; @@ -147,6 +163,26 @@ struct Value NixFloat fpoint; }; + // Returns the normal type of a Value. This only returns nThunk if the + // Value hasn't been forceValue'd + inline NormalType normalType() const + { + switch (type) { + case tInt: return nInt; + case tBool: return nBool; + case tString: return nString; + case tPath: return nPath; + case tNull: return nNull; + case tAttrs: return nAttrs; + case tList1: case tList2: case tListN: return nList; + case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; + case tExternal: return nExternal; + case tFloat: return nFloat; + case tThunk: case tApp: case tBlackhole: return nThunk; + } + abort(); + } + bool isList() const { return type == tList1 || type == tList2 || type == tListN; From 9f056f7afdb85b8c3bd59638197e356f269129b2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 00:19:05 +0100 Subject: [PATCH 200/477] Introduce Value type setters and make use of them --- src/libexpr/attr-set.cc | 2 +- src/libexpr/eval-inline.hh | 4 ++-- src/libexpr/eval.cc | 24 ++++++++++++------------ src/libexpr/value.hh | 35 +++++++++++++++++++++++++++-------- src/nix/repl.cc | 2 +- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b1d61a285..17886a426 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -25,7 +25,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) return; } clearValue(v); - v.type = tAttrs; + v.setAttrs(); v.attrs = allocBindings(capacity); nrAttrsets++; nrAttrsInAttrsets += capacity; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 30f6ec7db..a0fd9b569 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -36,11 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos) Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.type = tBlackhole; + v.setBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.type = tThunk; + v.setThunk(); v.thunk.env = env; v.thunk.expr = expr; throw; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 48fe0bbda..f68828944 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -431,7 +431,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) } clearValue(vEmptySet); - vEmptySet.type = tAttrs; + vEmptySet.setAttrs(); vEmptySet.attrs = allocBindings(0); createBaseEnv(); @@ -548,7 +548,7 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; + vPrimOp->setPrimOp(); vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; Value v; mkApp(v, *vPrimOp, *vPrimOp); @@ -556,7 +556,7 @@ Value * EvalState::addPrimOp(const string & name, } Value * v = allocValue(); - v->type = tPrimOp; + v->setPrimOp(); v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; @@ -572,7 +572,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; + vPrimOp->setPrimOp(); vPrimOp->primOp = new PrimOp(std::move(primOp)); Value v; mkApp(v, *vPrimOp, *vPrimOp); @@ -584,7 +584,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) primOp.name = symbols.create(std::string(primOp.name, 2)); Value * v = allocValue(); - v->type = tPrimOp; + v->setPrimOp(); v->primOp = new PrimOp(std::move(primOp)); staticBaseEnv.vars[envName] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; @@ -714,7 +714,7 @@ void mkString(Value & v, const char * s) Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.type = tString; + v.setString(); v.string.s = dupStringWithLen(s.data(), s.size()); v.string.context = 0; if (!context.empty()) { @@ -794,11 +794,11 @@ void EvalState::mkList(Value & v, size_t size) { clearValue(v); if (size == 1) - v.type = tList1; + v.setList1(); else if (size == 2) - v.type = tList2; + v.setList2(); else { - v.type = tListN; + v.setListN(); v.bigList.size = size; v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; } @@ -810,7 +810,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.type = tThunk; + v.setThunk(); v.thunk.env = &env; v.thunk.expr = expr; nrThunks++; @@ -1207,7 +1207,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.type = tLambda; + v.setLambda(); v.lambda.env = &env; v.lambda.fun = this; } @@ -1252,7 +1252,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.type = tPrimOpApp; + v.setPrimOpApp(); v.primOpApp.left = fun2; v.primOpApp.right = &arg; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 833af0f3d..0995dcd7b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -107,6 +107,25 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { ValueType type; + + inline void setInt() { type = tInt; }; + inline void setBool() { type = tBool; }; + inline void setString() { type = tString; }; + inline void setPath() { type = tPath; }; + inline void setNull() { type = tNull; }; + inline void setAttrs() { type = tAttrs; }; + inline void setList1() { type = tList1; }; + inline void setList2() { type = tList2; }; + inline void setListN() { type = tListN; }; + inline void setThunk() { type = tThunk; }; + inline void setApp() { type = tApp; }; + inline void setLambda() { type = tLambda; }; + inline void setBlackhole() { type = tBlackhole; }; + inline void setPrimOp() { type = tPrimOp; }; + inline void setPrimOpApp() { type = tPrimOpApp; }; + inline void setExternal() { type = tExternal; }; + inline void setFloat() { type = tFloat; }; + union { NixInt integer; @@ -223,7 +242,7 @@ static inline void clearValue(Value & v) static inline void mkInt(Value & v, NixInt n) { clearValue(v); - v.type = tInt; + v.setInt(); v.integer = n; } @@ -231,7 +250,7 @@ static inline void mkInt(Value & v, NixInt n) static inline void mkFloat(Value & v, NixFloat n) { clearValue(v); - v.type = tFloat; + v.setFloat(); v.fpoint = n; } @@ -239,7 +258,7 @@ static inline void mkFloat(Value & v, NixFloat n) static inline void mkBool(Value & v, bool b) { clearValue(v); - v.type = tBool; + v.setBool(); v.boolean = b; } @@ -247,13 +266,13 @@ static inline void mkBool(Value & v, bool b) static inline void mkNull(Value & v) { clearValue(v); - v.type = tNull; + v.setNull(); } static inline void mkApp(Value & v, Value & left, Value & right) { - v.type = tApp; + v.setApp(); v.app.left = &left; v.app.right = &right; } @@ -261,7 +280,7 @@ static inline void mkApp(Value & v, Value & left, Value & right) static inline void mkPrimOpApp(Value & v, Value & left, Value & right) { - v.type = tPrimOpApp; + v.setPrimOpApp(); v.app.left = &left; v.app.right = &right; } @@ -269,7 +288,7 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right) static inline void mkStringNoCopy(Value & v, const char * s) { - v.type = tString; + v.setString(); v.string.s = s; v.string.context = 0; } @@ -287,7 +306,7 @@ void mkString(Value & v, const char * s); static inline void mkPathNoCopy(Value & v, const char * s) { clearValue(v); - v.type = tPath; + v.setPath(); v.path = s; } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 71794a309..3cee81b49 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -551,7 +551,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.type = tThunk; + v.setThunk(); v.thunk.env = env; v.thunk.expr = e; addVarToScope(state->symbols.create(name), v); From 22ead43a0b8f94f5a4fb64cff14bf6a2a35d671c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:09:10 +0100 Subject: [PATCH 201/477] Use Value::normalType on all forced values instead of Value::type --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 22 ++--- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 112 +++++++++++++------------- src/libexpr/flake/flake.cc | 12 +-- src/libexpr/get-drvs.cc | 30 +++---- src/libexpr/primops.cc | 89 +++++++++----------- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 12 +-- src/libexpr/value-to-json.cc | 25 +++--- src/libexpr/value-to-xml.cc | 33 ++++---- src/nix-env/nix-env.cc | 16 ++-- src/nix/eval.cc | 4 +- src/nix/flake.cc | 2 +- src/nix/repl.cc | 47 +++++------ 15 files changed, 199 insertions(+), 213 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 83854df49..54e13e6a2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -67,7 +67,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (apType == apAttr) { - if (v->type != tAttrs) + if (v->normalType() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a11327f77..3c97f1201 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -390,14 +390,14 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.type == tString) + if (v.normalType() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == tPath) + else if (v.normalType() == nPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; - else if (v.type == tBool) + else if (v.normalType() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.type == tAttrs) + else if (v.normalType() == nAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; @@ -442,7 +442,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro auto & v = forceValue(); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); @@ -512,10 +512,10 @@ std::string AttrCursor::getString() auto & v = forceValue(); - if (v.type != tString && v.type != tPath) + if (v.normalType() != nString && v.normalType() != nPath) throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); - return v.type == tString ? v.string.s : v.path; + return v.normalType() == nString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() @@ -543,9 +543,9 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type == tString) + if (v.normalType() == nString) return {v.string.s, v.getContext()}; - else if (v.type == tPath) + else if (v.normalType() == nPath) return {v.path, {}}; else throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); @@ -567,7 +567,7 @@ bool AttrCursor::getBool() auto & v = forceValue(); - if (v.type != tBool) + if (v.normalType() != nBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; @@ -589,7 +589,7 @@ std::vector AttrCursor::getAttrs() auto & v = forceValue(); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector attrs; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index a0fd9b569..9b644d5cb 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f68828944..f33426b59 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.type == tNull) + if (nameVal.normalType() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->type != tAttrs || + if (vAttrs->normalType() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->type != tAttrs || + if (vAttrs->normalType() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po return; } - if (fun.type == tAttrs) { + if (fun.normalType() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.type == tAttrs) { + if (fun.normalType() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - ValueType firstType = tString; + NormalType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1573,36 +1573,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.type; + firstType = vTmp.normalType(); first = false; } - if (firstType == tInt) { - if (vTmp.type == tInt) { + if (firstType == nInt) { + if (vTmp.normalType() == nInt) { n += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.normalType() == nFloat) { // Upgrade the type from int to float; - firstType = tFloat; + firstType = nFloat; nf = n; nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); - } else if (firstType == tFloat) { - if (vTmp.type == tInt) { + } else if (firstType == nFloat) { + if (vTmp.normalType() == nInt) { nf += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.normalType() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else - s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + s << state.coerceToString(pos, vTmp, context, false, firstType == nString); } - if (firstType == tInt) + if (firstType == nInt) mkInt(v, n); - else if (firstType == tFloat) + else if (firstType == nFloat) mkFloat(v, nf); - else if (firstType == tPath) { + else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); @@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.type == tAttrs) { + if (v.normalType() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tInt) + if (v.normalType() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type == tInt) + if (v.normalType() == nInt) return v.integer; - else if (v.type != tFloat) + else if (v.normalType() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) + if (v.normalType() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tString) { + if (v.normalType() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.type != tAttrs) return false; + if (v.normalType() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->type != tString) return false; + if (i->value->normalType() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string s; - if (v.type == tString) { + if (v.normalType() == nString) { copyContext(v, context); return v.string.s; } - if (v.type == tPath) { + if (v.normalType() == nPath) { Path path(canonPath(v.path)); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.type == tAttrs) { + if (v.normalType() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.type == tExternal) + if (v.normalType() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return std::to_string(v.integer); - if (v.type == tFloat) return std::to_string(v.fpoint); - if (v.type == tNull) return ""; + if (v.normalType() == nBool && v.boolean) return "1"; + if (v.normalType() == nBool && !v.boolean) return ""; + if (v.normalType() == nInt) return std::to_string(v.integer); + if (v.normalType() == nFloat) return std::to_string(v.fpoint); + if (v.normalType() == nNull) return ""; if (v.isList()) { string result; @@ -1884,40 +1884,38 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.type == tInt && v2.type == tFloat) + if (v1.normalType() == nInt && v2.normalType() == nFloat) return v1.integer == v2.fpoint; - if (v1.type == tFloat && v2.type == tInt) + if (v1.normalType() == nFloat && v2.normalType() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.type != v2.type) return false; + if (v1.normalType() != v2.normalType()) return false; - switch (v1.type) { + switch (v1.normalType()) { - case tInt: + case nInt: return v1.integer == v2.integer; - case tBool: + case nBool: return v1.boolean == v2.boolean; - case tString: + case nString: return strcmp(v1.string.s, v2.string.s) == 0; - case tPath: + case nPath: return strcmp(v1.path, v2.path) == 0; - case tNull: + case nNull: return true; - case tList1: - case tList2: - case tListN: + case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; - case tAttrs: { + case nAttrs: { /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { @@ -1939,15 +1937,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) } /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: + case nFunction: return false; - case tExternal: + case nExternal: return *v1.external == *v2.external; - case tFloat: + case nFloat: return v1.fpoint == v2.fpoint; default: diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 881b1b4e5..c126b2c40 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type == tString) + if (attr.value->normalType() == nString) attrs.emplace(attr.name, attr.value->string.s); else throw TypeError("flake input attribute '%s' is %s while a string is expected", @@ -235,17 +235,17 @@ static Flake getFlake( for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); - if (setting.value->type == tString) + if (setting.value->normalType() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); - else if (setting.value->type == tInt) + else if (setting.value->normalType() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); - else if (setting.value->type == tBool) + else if (setting.value->normalType() == nBool) flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); - else if (setting.value->isList()) { + else if (setting.value->normalType() == nList) { std::vector ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { auto elem = setting.value->listElems()[n]; - if (elem->type != tString) + if (elem->normalType() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 91916e8bf..93788273f 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type != tString) throw errMsg; + if ((*i)->normalType() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.isList()) { + if (v.normalType() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.type == tInt || v.type == tBool || v.type == tString || - v.type == tFloat; + else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString || + v.normalType() == nFloat; } @@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; + if (!v || v->normalType() != nString) return ""; return v->string.s; } @@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { + if (v->normalType() == nInt) return v->integer; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ NixInt n; @@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tFloat) return v->fpoint; - if (v->type == tString) { + if (v->normalType() == nFloat) return v->fpoint; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ NixFloat n; @@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { + if (v->normalType() == nBool) return v->boolean; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->type == tAttrs) { + if (i->value->normalType() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.isList()) { + else if (v.normalType() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..d501f7482 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -356,24 +356,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList1: case tList2: case tListN: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - case tExternal: + switch (args[0]->normalType()) { + case nInt: t = "int"; break; + case nBool: t = "bool"; break; + case nString: t = "string"; break; + case nPath: t = "path"; break; + case nNull: t = "null"; break; + case nAttrs: t = "set"; break; + case nList: t = "list"; break; + case nFunction: t = "lambda"; break; + case nExternal: t = args[0]->external->typeOf(); break; - case tFloat: t = "float"; break; - default: abort(); + case nFloat: t = "float"; break; + case nThunk: abort(); } mkString(v, state.symbols.create(t)); } @@ -393,7 +389,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tNull); + mkBool(v, args[0]->normalType() == nNull); } static RegisterPrimOp primop_isNull({ @@ -413,18 +409,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - bool res; - switch (args[0]->type) { - case tLambda: - case tPrimOp: - case tPrimOpApp: - res = true; - break; - default: - res = false; - break; - } - mkBool(v, res); + mkBool(v, args[0]->normalType() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -440,7 +425,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tInt); + mkBool(v, args[0]->normalType() == nInt); } static RegisterPrimOp primop_isInt({ @@ -456,7 +441,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tFloat); + mkBool(v, args[0]->normalType() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -472,7 +457,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tString); + mkBool(v, args[0]->normalType() == nString); } static RegisterPrimOp primop_isString({ @@ -488,7 +473,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tBool); + mkBool(v, args[0]->normalType() == nBool); } static RegisterPrimOp primop_isBool({ @@ -504,7 +489,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tPath); + mkBool(v, args[0]->normalType() == nPath); } static RegisterPrimOp primop_isPath({ @@ -520,20 +505,20 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->type == tFloat && v2->type == tInt) + if (v1->normalType() == nFloat && v2->normalType() == nInt) return v1->fpoint < v2->integer; - if (v1->type == tInt && v2->type == tFloat) + if (v1->normalType() == nInt && v2->normalType() == nFloat) return v1->integer < v2->fpoint; - if (v1->type != v2->type) + if (v1->normalType() != v2->normalType()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->type) { - case tInt: + switch (v1->normalType()) { + case nInt: return v1->integer < v2->integer; - case tFloat: + case nFloat: return v1->fpoint < v2->fpoint; - case tString: + case nString: return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: + case nPath: return strcmp(v1->path, v2->path) < 0; default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); @@ -777,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tString) + if (args[0]->normalType() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -902,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->type == tNull) continue; + if (i->value->normalType() == nNull) continue; } if (i->name == state.sContentAddressed) { @@ -1308,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1808,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args }); state.forceValue(*args[0], pos); - if (args[0]->type != tLambda) + if (args[0]->normalType() != nFunction) throw TypeError({ .hint = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", @@ -2074,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tAttrs); + mkBool(v, args[0]->normalType() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2337,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->isList()); + mkBool(v, args[0]->normalType() == nList); } static RegisterPrimOp primop_isList({ @@ -2831,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); @@ -2850,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); @@ -2869,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); @@ -2896,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - if (args[0]->type == tFloat || args[1]->type == tFloat) { + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a77035c16..2461ebc99 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..6d93e1dc2 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,25 +85,25 @@ static void fetchTree( state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tPath || attr.value->type == tString) + if (attr.value->normalType() == nPath || attr.value->normalType() == nString) addURI( state, attrs, attr.name, state.coerceToString(*attr.pos, *attr.value, context, false, false) ); - else if (attr.value->type == tString) + else if (attr.value->normalType() == nString) addURI(state, attrs, attr.name, attr.value->string.s); - else if (attr.value->type == tBool) + else if (attr.value->normalType() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->type == tInt) + else if (attr.value->normalType() == nInt) attrs.emplace(attr.name, attr.value->integer); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", @@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 6ec8315ba..b5f4c8654 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: out.write(v.integer); break; - case tBool: + case nBool: out.write(v.boolean); break; - case tString: + case nString: copyContext(v, context); out.write(v.string.s); break; - case tPath: + case nPath: out.write(state.copyPathToStore(context, v.path)); break; - case tNull: + case nNull: out.write(nullptr); break; - case tAttrs: { + case nAttrs: { auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); if (maybeString) { out.write(*maybeString); @@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tList1: case tList2: case tListN: { + case nList: { auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); @@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tExternal: + case nExternal: v.external->printValueAsJSON(state, strict, out, context); break; - case tFloat: + case nFloat: out.write(v.fpoint); break; - default: + case nThunk: + throw TypeError("cannot convert %1% to JSON", showType(v)); + + case nFunction: throw TypeError("cannot convert %1% to JSON", showType(v)); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1f0b1541d..26be07cff 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); break; - case tBool: + case nBool: doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); break; - case tString: + case nString: /* !!! show the context? */ copyContext(v, context); doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); break; - case tPath: + case nPath: doc.writeEmptyElement("path", singletonAttrs("value", v.path)); break; - case tNull: + case nNull: doc.writeEmptyElement("null"); break; - case tAttrs: + case nAttrs: if (state.isDerivation(v)) { XMLAttrs xmlAttrs; @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->normalType() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->normalType() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; - case tList1: case tList2: case tListN: { + case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); break; } - case tLambda: { + case nFunction: { + if (!v.isLambda()) { + // FIXME: Serialize primops and primopapps + doc.writeEmptyElement("unevaluated"); + break; + } XMLAttrs xmlAttrs; if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); @@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } - case tExternal: + case nExternal: v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); break; - case tFloat: + case nFloat: doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); break; - default: + case nThunk: doc.writeEmptyElement("unevaluated"); } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a4b5c9e2c..404fd5111 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) i.queryName(), j) }); else { - if (v->type == tString) { + if (v->normalType() == nString) { attrs2["type"] = "string"; attrs2["value"] = v->string.s; xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tInt) { + } else if (v->normalType() == nInt) { attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tFloat) { + } else if (v->normalType() == nFloat) { attrs2["type"] = "float"; attrs2["value"] = (format("%1%") % v->fpoint).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tBool) { + } else if (v->normalType() == nBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->isList()) { + } else if (v->normalType() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type != tString) continue; + if (v->listElems()[j]->normalType() != nString) continue; XMLAttrs attrs3; attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } - } else if (v->type == tAttrs) { + } else if (v->normalType() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); Bindings & attrs = *v->attrs; for (auto &i : attrs) { Attr & a(*attrs.find(i.name)); - if(a.value->type != tString) continue; + if(a.value->normalType() != nString) continue; XMLAttrs attrs3; attrs3["type"] = i.name; attrs3["value"] = a.value->string.s; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 0f02919de..bba3b1bc6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand recurse = [&](Value & v, const Pos & pos, const Path & path) { state->forceValue(v); - if (v.type == tString) + if (v.normalType() == nString) // FIXME: disallow strings with contexts? writeFile(path, v.string.s); - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7a7c71676..80b050091 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand if (v.type == tLambda) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type == tAttrs) { + } else if (v.normalType() == nAttrs) { for (auto & attr : *v.attrs) try { state->forceValue(*attr.value, *attr.pos); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 3cee81b49..56184efb9 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -446,7 +446,7 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.type == tPath || v.type == tString) { + if (v.normalType() == nPath || v.normalType() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); @@ -669,31 +669,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; break; - case tBool: + case nBool: str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; break; - case tString: + case nString: str << ANSI_YELLOW; printStringValue(str, v.string.s); str << ANSI_NORMAL; break; - case tPath: + case nPath: str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? break; - case tNull: + case nNull: str << ANSI_CYAN "null" ANSI_NORMAL; break; - case tAttrs: { + case nAttrs: { seen.insert(&v); bool isDrv = state->isDerivation(v); @@ -738,9 +738,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; } - case tList1: - case tList2: - case tListN: + case nList: seen.insert(&v); str << "[ "; @@ -761,22 +759,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "]"; break; - case tLambda: { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - break; - } - - case tPrimOp: - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + case nFunction: + if (v.type == tLambda) { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; + } else if (v.type == tPrimOp) { + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + } else if (v.type == tPrimOpApp) { + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + } else { + abort(); + } break; - case tPrimOpApp: - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; - break; - - case tFloat: + case nFloat: str << v.fpoint; break; From bf9890396731a2bbe4f04a49684dee463d818906 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:15:11 +0100 Subject: [PATCH 202/477] Add ValueType checking functions for types that have the same NormalType --- src/libexpr/eval-inline.hh | 6 +++--- src/libexpr/eval.cc | 20 ++++++++++---------- src/libexpr/flake/flake.cc | 4 ++-- src/libexpr/primops.cc | 6 +++--- src/libexpr/value.hh | 14 ++++++++++++++ src/nix/flake.cc | 6 +++--- src/nix/main.cc | 2 +- src/nix/repl.cc | 8 ++++---- 8 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 9b644d5cb..8c40c2565 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -32,7 +32,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const void EvalState::forceValue(Value & v, const Pos & pos) { - if (v.type == tThunk) { + if (v.isThunk()) { Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { @@ -46,9 +46,9 @@ void EvalState::forceValue(Value & v, const Pos & pos) throw; } } - else if (v.type == tApp) + else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.type == tBlackhole) + else if (v.isBlackhole()) throwEvalError(pos, "infinite recursion encountered"); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f33426b59..5f9d19b8d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -158,10 +158,10 @@ std::ostream & operator << (std::ostream & str, const Value & v) const Value *getPrimOp(const Value &v) { const Value * primOp = &v; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); return primOp; } @@ -601,9 +601,9 @@ Value & EvalState::getBuiltin(const string & name) std::optional EvalState::getDoc(Value & v) { - if (v.type == tPrimOp || v.type == tPrimOpApp) { + if (v.isPrimOp() || v.isPrimOpApp()) { auto v2 = &v; - while (v2->type == tPrimOpApp) + while (v2->isPrimOpApp()) v2 = v2->primOpApp.left; if (v2->primOp->doc) return Doc { @@ -1227,11 +1227,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* Figure out the number of arguments still needed. */ size_t argsDone = 0; Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { argsDone++; primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); auto arity = primOp->primOp->arity; auto argsLeft = arity - argsDone; @@ -1242,7 +1242,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) Value * vArgs[arity]; auto n = arity - 1; vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; /* And call the primop. */ @@ -1264,7 +1264,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po forceValue(fun, pos); - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + if (fun.isPrimOp() || fun.isPrimOpApp()) { callPrimOp(fun, arg, v, pos); return; } @@ -1285,7 +1285,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } } - if (fun.type != tLambda) + if (!fun.isLambda()) throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1378,7 +1378,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { res = fun; return; } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c126b2c40..2f9658ab8 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -73,7 +73,7 @@ static std::tuple fetchOrSubstituteTree( static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) { - if (value.type == tThunk && value.isTrivial()) + if (value.isThunk() && value.isTrivial()) state.forceValue(value, pos); } @@ -216,7 +216,7 @@ static Flake getFlake( if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d501f7482..f6ca612f4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2239,11 +2239,11 @@ static RegisterPrimOp primop_catAttrs({ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { state.mkAttrs(v, 0); return; } - if (args[0]->type != tLambda) + if (!args[0]->isLambda()) throw TypeError({ .hint = hintfmt("'functionArgs' requires a function"), .errPos = pos @@ -2674,7 +2674,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) return CompareValues()(a, b); Value vTmp1, vTmp2; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 0995dcd7b..e743da9c3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -126,6 +126,20 @@ struct Value inline void setExternal() { type = tExternal; }; inline void setFloat() { type = tFloat; }; + // Functions needed to distinguish the type + // These should be removed eventually, by putting the functionality that's + // needed by callers into methods of this type + + // normalType() == nThunk + inline bool isThunk() const { return type == tThunk; }; + inline bool isApp() const { return type == tApp; }; + inline bool isBlackhole() const { return type == tBlackhole; }; + + // normalType() == nFunction + inline bool isLambda() const { return type == tLambda; }; + inline bool isPrimOp() const { return type == tPrimOp; }; + inline bool isPrimOpApp() const { return type == tPrimOpApp; }; + union { NixInt integer; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 80b050091..e4da0348c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -260,7 +260,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); if (!body || body->matchAttrs || std::string(body->arg) != "prev") @@ -276,7 +276,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type == tLambda) { + if (v.isLambda()) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); } else if (v.normalType() == nAttrs) { @@ -371,7 +371,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda) + if (!v.isLambda()) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || diff --git a/src/nix/main.cc b/src/nix/main.cc index 27b1d7257..e7a15dec9 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -272,7 +272,7 @@ void mainWrapped(int argc, char * * argv) auto builtins = state.baseEnv.values[0]->attrs; for (auto & builtin : *builtins) { auto b = nlohmann::json::object(); - if (builtin.value->type != tPrimOp) continue; + if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp; if (!primOp->doc) continue; b["arity"] = primOp->arity; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 56184efb9..047e2dc59 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -450,7 +450,7 @@ bool NixRepl::processLine(string line) PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); - } else if (v.type == tLambda) { + } else if (v.isLambda()) { pos = v.lambda.fun->pos; } else { // assume it's a derivation @@ -760,13 +760,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nFunction: - if (v.type == tLambda) { + if (v.isLambda()) { std::ostringstream s; s << v.lambda.fun->pos; str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - } else if (v.type == tPrimOp) { + } else if (v.isPrimOp()) { str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - } else if (v.type == tPrimOpApp) { + } else if (v.isPrimOpApp()) { str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; } else { abort(); From 730b152b190135adef2f53c7a80cfd1111d37ead Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:22:58 +0100 Subject: [PATCH 203/477] Make Value::type private This is an implementation detail and shouldn't be used. Use normalType() and the various is functions instead --- src/libexpr/eval.cc | 2 +- src/libexpr/value.hh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5f9d19b8d..1d11039ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -68,7 +68,7 @@ RootValue allocRootValue(Value * v) } -static void printValue(std::ostream & str, std::set & active, const Value & v) +void printValue(std::ostream & str, std::set & active, const Value & v) { checkInterrupt(); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index e743da9c3..4050d7e4b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -106,8 +106,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { +private: ValueType type; +friend std::string showType(const Value & v); +friend void printValue(std::ostream & str, std::set & active, const Value & v); + +public: + inline void setInt() { type = tInt; }; inline void setBool() { type = tBool; }; inline void setString() { type = tString; }; From f890830b337f321f17ad36b45d7d63801a753554 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 11 Dec 2020 16:31:14 +0100 Subject: [PATCH 204/477] primops/fromJSON: add error position in case of parse error This makes it easier to track down where invalid JSON was passed to `builtins.fromJSON`. --- src/libexpr/primops.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..63624c520 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1621,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) { string s = state.forceStringNoCtx(*args[0], pos); - parseJSON(state, s, v); + try { + parseJSON(state, s, v); + } catch (JSONParseError &e) { + e.addTrace(pos, "while decoding a JSON string"); + throw e; + } } static RegisterPrimOp primop_fromJSON({ From 44c3fbc6e03ec518f6174c2b7c21b603973beb91 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:03:54 +0200 Subject: [PATCH 205/477] Fix `addTextToStore` for binary caches Because of a too eager refactoring, `addTextToStore` used to throw an error because the input wasn't a valid nar. Partially revert that refactoring to wrap the text into a proper nar (using `dumpString`) to make this method work again --- src/libstore/binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5b081c1ae..94c11355f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s if (!repair && isValidPath(path)) return path; - auto source = StringSource { s }; + StringSink sink; + dumpString(s, sink); + auto source = StringSource { *sink.s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { path, nar.first }; info.narSize = nar.second; From 7080321618e29033a8b5dc2f9fc938dcf2df270d Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 10:54:24 +0100 Subject: [PATCH 206/477] Use the fs accessor for readInvalidDerivation Extend `FSAccessor::readFile` to allow not checking that the path is a valid one, and rewrite `readInvalidDerivation` using this extended `readFile`. Several places in the code use `readInvalidDerivation`, either because they need to read a derivation that has been written in the store but not registered yet, or more generally to prevent a deadlock because `readDerivation` tries to lock the state, so can't be called from a place where the lock is already held. However, `readInvalidDerivation` implicitely assumes that the store is a `LocalFSStore`, which isn't always the case. The concrete motivation for this is that it's required for `nix copy --from someBinaryCache` to work, which is tremendously useful for the tests. --- src/libstore/fs-accessor.hh | 9 ++++++++- src/libstore/local-fs-store.cc | 8 ++++---- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.hh | 4 ++-- src/libstore/store-api.cc | 21 +++++++++------------ 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 64780a6da..c825e84f2 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -25,7 +25,14 @@ public: virtual StringSet readDirectory(const Path & path) = 0; - virtual std::string readFile(const Path & path) = 0; + /** + * Read a file inside the store. + * + * If `requireValidPath` is set to `true` (the default), the path must be + * inside a valid store path, otherwise it just needs to be physically + * present (but not necessarily properly registered) + */ + virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; virtual std::string readLink(const Path & path) = 0; }; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index e7c3dae92..6de13c73a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor LocalStoreAccessor(ref store) : store(store) { } - Path toRealPath(const Path & path) + Path toRealPath(const Path & path, bool requireValidPath = true) { auto storePath = store->toStorePath(path).first; - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { - return nix::readFile(toRealPath(path)); + return nix::readFile(toRealPath(path, requireValidPath)); } std::string readLink(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1427a0f98..784ebb719 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 63bde92de..f43456f0b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & } } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) +std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) { auto path = canonPath(path_); auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); auto i = nars.find(std::string(storePath.hashPart())); @@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path) return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path) +std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) { - auto res = fetch(path); + auto res = fetch(path, requireValidPath); return res.first->readFile(res.second); } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 347cf5764..594852d0e 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor Path cacheDir; - std::pair, Path> fetch(const Path & path_); + std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); friend class BinaryCacheStore; @@ -32,7 +32,7 @@ public: StringSet readDirectory(const Path & path) override; - std::string readFile(const Path & path) override; + std::string readFile(const Path & path, bool requireValidPath = true) override; std::string readLink(const Path & path) override; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..25e28cffa 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1018,26 +1018,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } - -Derivation Store::readDerivation(const StorePath & drvPath) +Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) { - auto accessor = getFSAccessor(); + auto accessor = store.getFSAccessor(); try { - return parseDerivation(*this, - accessor->readFile(printStorePath(drvPath)), + return parseDerivation(store, + accessor->readFile(store.printStorePath(drvPath), requireValidPath), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } } +Derivation Store::readDerivation(const StorePath & drvPath) +{ return readDerivationCommon(*this, drvPath, true); } + Derivation Store::readInvalidDerivation(const StorePath & drvPath) -{ - return parseDerivation( - *this, - readFile(Store::toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)); -} +{ return readDerivationCommon(*this, drvPath, false); } } From 6e899278d305da904fb766937f56344841c022b3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 6 Nov 2020 09:51:17 +0100 Subject: [PATCH 207/477] Better detect when `buildPaths` would be a no-op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildPaths` can be called even for stores where it's not defined in case it's bound to be a no-op. The “no-op detection” mechanism was only detecting the case wher `buildPaths` was called on a set of (non-drv) paths that were already present on the store. This commit extends this mechanism to also detect the case where `buildPaths` is called on a set of derivation outputs which are already built on the store. This only works with the ca-derivations flag. It could be possible to extend this to also work without it, but it would add quite a bit of complexity, and it's not used without it anyways. --- src/libstore/store-api.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..50905bb33 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -729,9 +729,17 @@ void Store::buildPaths(const std::vector & paths, BuildMod StorePathSet paths2; for (auto & path : paths) { - if (path.path.isDerivation()) - unsupported("buildPaths"); - paths2.insert(path.path); + if (path.path.isDerivation()) { + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & outputName : path.outputs) { + if (!queryRealisation({path.path, outputName})) + unsupported("buildPaths"); + } + } else + unsupported("buildPaths"); + + } else + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) From 962b82ef25069893779ed56d31e44814793f9273 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:34:45 +0100 Subject: [PATCH 208/477] Fix BinaryCacheStore::registerDrvOutput Was crashing because coercing a json document into a string is only valid if the json is a string, otherwise we need to call `.dump()` --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 94c11355f..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -460,7 +460,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.toJSON(), "application/json"); + upsertFile(filePath, info.toJSON().dump(), "application/json"); } ref BinaryCacheStore::getFSAccessor() From cac8d5b742ec0cb80ad7232e20f63c74a217e545 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 16 Dec 2020 13:36:17 +0100 Subject: [PATCH 209/477] Don't ignore an absent drv file in queryPartialDrvOutputMap This ignore was here because `queryPartialDrvOutputMap` was used both 1. as a cache to avoid having to re-read the derivation (when gc-ing for example), and 2. as the source of truth for ca realisations The use-case 2. required it to be able to work even when the derivation wasn't there anymore (see https://github.com/NixOS/nix/issues/4138). However, this use-case is now handled by `queryRealisation`, meaning that we can safely error out if the derivation isn't there anymore --- src/libstore/local-store.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1539c94e2..20bbc73cf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -912,12 +912,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto state(_state.lock()); std::map> outputs; uint64_t drvId; - try { - drvId = queryValidPathId(*state, path); - } catch (InvalidPath&) { - // Ignore non-existing drvs as they might still have an output map - // defined if ca-derivations is enabled - } + drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); while (use.next()) outputs.insert_or_assign( From 4d458394991f3086c3c9c306d000e6c0058c4fa7 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 17 Dec 2020 11:35:24 +0100 Subject: [PATCH 210/477] Fix the detection of already built drv outputs PRs #4370 and #4348 had a bad interaction in that the second broke the fist one in a not trivial way. The issue was that since #4348 the logic for detecting whether a derivation output is already built requires some logic that was specific to the `LocalStore`. It happens though that most of this logic could be upstreamed to any `Store`, which is what this commit does. --- src/libstore/derivations.cc | 32 +++++++++++++++++++++++++- src/libstore/derivations.hh | 4 ++++ src/libstore/local-store.cc | 45 ++++--------------------------------- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 39 +++++++++++++++++++++++++------- src/libstore/store-api.hh | 9 ++++++-- 6 files changed, 78 insertions(+), 53 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5bcc7f012..7466c7d41 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) { +std::optional Derivation::tryResolveUncached(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -771,4 +771,34 @@ std::optional Derivation::tryResolve(Store & store) { return resolved; } +std::optional Derivation::tryResolve(Store& store) +{ + auto drvPath = writeDerivation(store, *this, NoRepair, false); + return Derivation::tryResolve(store, drvPath); +} + +std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) +{ + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; + + { + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(drvPath); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; + } + } + + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(drvPath); + auto attempt = drv.tryResolveUncached(store); + if (!attempt) + return std::nullopt; + /* Store in memo table. */ + resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); + return *attempt; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4e5985fab..3d8f19aef 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,10 +138,14 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); + static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + +private: + std::optional tryResolveUncached(Store & store); }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 20bbc73cf..e9f9bde4d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -877,35 +877,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } -// Try to resolve the derivation at path `original`, with a caching layer -// to make it more efficient -std::optional cachedResolve( - LocalStore& store, - const StorePath& original) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(original); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(original); - auto attempt = drv.tryResolve(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(original, *attempt); - return *attempt; -} std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { @@ -924,20 +898,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) if (!settings.isExperimentalFeatureEnabled("ca-derivations")) return outputs; - auto drv = readDerivation(path); - - auto resolvedDrv = cachedResolve(*this, path); - - if (!resolvedDrv) { - for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { - if (!outputs.count(outputName)) - outputs.emplace(outputName, std::nullopt); - } - return outputs; - } - - auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); - for (auto& [outputName, hash] : resolvedDrvHashes) { + auto drv = readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto& [outputName, hash] : drvHashes) { auto realisation = queryRealisation(DrvOutput{hash, outputName}); if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69559e346..877dba742 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 50905bb33..2cd39ab11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +{ + std::map> outputs; + auto drv = readInvalidDerivation(path); + for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + outputs.emplace(outputName, output.second); + } + return outputs; +} + +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +{ + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = Derivation::tryResolve(*this, path); + if (resolvedDrv) { + auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); + if (isValidPath(resolvedDrvPath)) + return queryDerivationOutputMapNoResolve(resolvedDrvPath); + } + } + return queryDerivationOutputMapNoResolve(path); +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; @@ -730,14 +753,14 @@ void Store::buildPaths(const std::vector & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto & outputName : path.outputs) { - if (!queryRealisation({path.path, outputName})) - unsupported("buildPaths"); - } - } else - unsupported("buildPaths"); - + auto outPaths = queryPartialDerivationOutputMap(path.path); + for (auto & outputName : path.outputs) { + auto currentOutputPathIter = outPaths.find(outputName); + if (currentOutputPathIter == outPaths.end() || + !currentOutputPathIter->second || + !isValidPath(*currentOutputPathIter->second)) + unsupported("buildPaths"); + } } else paths2.insert(path.path); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7cdadc1f3..ce95b78b1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -416,8 +416,13 @@ public: /* Query the mapping outputName => outputPath for the given derivation. All outputs are mentioned so ones mising the mapping are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path) - { unsupported("queryPartialDerivationOutputMap"); } + virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + + /* + * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve + * the derivation + */ + virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ From d67e02919c7f941615407dfd14cfdab6a28c4c26 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 17 Dec 2020 14:42:52 +0100 Subject: [PATCH 211/477] Rename ValueType -> InternalType, NormalType -> ValueType And Value::type to Value::internalType, such that type() can be used in the next commit to get the new ValueType --- src/libexpr/eval.cc | 16 +++++----- src/libexpr/eval.hh | 2 +- src/libexpr/flake/flake.cc | 2 +- src/libexpr/value.hh | 64 +++++++++++++++++++------------------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d11039ad..e14eb01c7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -77,7 +77,7 @@ void printValue(std::ostream & str, std::set & active, const Valu return; } - switch (v.type) { + switch (v.internalType) { case tInt: str << v.integer; break; @@ -165,7 +165,7 @@ const Value *getPrimOp(const Value &v) { return primOp; } -string showType(NormalType type) +string showType(ValueType type) { switch (type) { case nInt: return "an integer"; @@ -186,7 +186,7 @@ string showType(NormalType type) string showType(const Value & v) { - switch (v.type) { + switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); @@ -205,9 +205,9 @@ string showType(const Value & v) bool Value::isTrivial() const { return - type != tApp - && type != tPrimOpApp - && (type != tThunk + internalType != tApp + && internalType != tPrimOpApp + && (internalType != tThunk || (dynamic_cast(thunk.expr) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) || dynamic_cast(thunk.expr) @@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - NormalType firstType = nString; + ValueType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1728,7 +1728,7 @@ void copyContext(const Value & v, PathSet & context) std::vector> Value::getContext() { std::vector> res; - assert(type == tString); + assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) res.push_back(decodeContext(*p)); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 211529954..0e1f61baa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -346,7 +346,7 @@ private: /* Return a string representing the type of the value `v'. */ -string showType(NormalType type); +string showType(ValueType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair & active, const Value & v); public: - inline void setInt() { type = tInt; }; - inline void setBool() { type = tBool; }; - inline void setString() { type = tString; }; - inline void setPath() { type = tPath; }; - inline void setNull() { type = tNull; }; - inline void setAttrs() { type = tAttrs; }; - inline void setList1() { type = tList1; }; - inline void setList2() { type = tList2; }; - inline void setListN() { type = tListN; }; - inline void setThunk() { type = tThunk; }; - inline void setApp() { type = tApp; }; - inline void setLambda() { type = tLambda; }; - inline void setBlackhole() { type = tBlackhole; }; - inline void setPrimOp() { type = tPrimOp; }; - inline void setPrimOpApp() { type = tPrimOpApp; }; - inline void setExternal() { type = tExternal; }; - inline void setFloat() { type = tFloat; }; + inline void setInt() { internalType = tInt; }; + inline void setBool() { internalType = tBool; }; + inline void setString() { internalType = tString; }; + inline void setPath() { internalType = tPath; }; + inline void setNull() { internalType = tNull; }; + inline void setAttrs() { internalType = tAttrs; }; + inline void setList1() { internalType = tList1; }; + inline void setList2() { internalType = tList2; }; + inline void setListN() { internalType = tListN; }; + inline void setThunk() { internalType = tThunk; }; + inline void setApp() { internalType = tApp; }; + inline void setLambda() { internalType = tLambda; }; + inline void setBlackhole() { internalType = tBlackhole; }; + inline void setPrimOp() { internalType = tPrimOp; }; + inline void setPrimOpApp() { internalType = tPrimOpApp; }; + inline void setExternal() { internalType = tExternal; }; + inline void setFloat() { internalType = tFloat; }; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type // normalType() == nThunk - inline bool isThunk() const { return type == tThunk; }; - inline bool isApp() const { return type == tApp; }; - inline bool isBlackhole() const { return type == tBlackhole; }; + inline bool isThunk() const { return internalType == tThunk; }; + inline bool isApp() const { return internalType == tApp; }; + inline bool isBlackhole() const { return internalType == tBlackhole; }; // normalType() == nFunction - inline bool isLambda() const { return type == tLambda; }; - inline bool isPrimOp() const { return type == tPrimOp; }; - inline bool isPrimOpApp() const { return type == tPrimOpApp; }; + inline bool isLambda() const { return internalType == tLambda; }; + inline bool isPrimOp() const { return internalType == tPrimOp; }; + inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; union { @@ -204,9 +204,9 @@ public: // Returns the normal type of a Value. This only returns nThunk if the // Value hasn't been forceValue'd - inline NormalType normalType() const + inline ValueType normalType() const { - switch (type) { + switch (internalType) { case tInt: return nInt; case tBool: return nBool; case tString: return nString; @@ -224,22 +224,22 @@ public: bool isList() const { - return type == tList1 || type == tList2 || type == tListN; + return internalType == tList1 || internalType == tList2 || internalType == tListN; } Value * * listElems() { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } const Value * const * listElems() const { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } size_t listSize() const { - return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } /* Check whether forcing this value requires a trivial amount of From 12e65078ef5c511196c9e48f7fdf71f6c0e5c89f Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 17 Dec 2020 14:45:45 +0100 Subject: [PATCH 212/477] Rename Value::normalType() -> Value::type() --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 26 +++++----- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 74 +++++++++++++-------------- src/libexpr/flake/flake.cc | 16 +++--- src/libexpr/get-drvs.cc | 30 +++++------ src/libexpr/primops.cc | 44 ++++++++-------- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 12 ++--- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 +-- src/libexpr/value.hh | 6 +-- src/nix-env/nix-env.cc | 16 +++--- src/nix/eval.cc | 4 +- src/nix/flake.cc | 2 +- src/nix/repl.cc | 4 +- 16 files changed, 125 insertions(+), 125 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 54e13e6a2..2d37dcb7e 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -67,7 +67,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (apType == apAttr) { - if (v->normalType() != nAttrs) + if (v->type() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 3c97f1201..75e9af787 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -390,14 +390,14 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.normalType() == nString) + if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.normalType() == nPath) + else if (v.type() == nPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; - else if (v.normalType() == nBool) + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.normalType() == nAttrs) + else if (v.type() == nAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; @@ -442,7 +442,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro auto & v = forceValue(); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); @@ -512,10 +512,10 @@ std::string AttrCursor::getString() auto & v = forceValue(); - if (v.normalType() != nString && v.normalType() != nPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); + if (v.type() != nString && v.type() != nPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); - return v.normalType() == nString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() @@ -543,12 +543,12 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.normalType() == nString) + if (v.type() == nString) return {v.string.s, v.getContext()}; - else if (v.normalType() == nPath) + else if (v.type() == nPath) return {v.path, {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); } bool AttrCursor::getBool() @@ -567,7 +567,7 @@ bool AttrCursor::getBool() auto & v = forceValue(); - if (v.normalType() != nBool) + if (v.type() != nBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; @@ -589,7 +589,7 @@ std::vector AttrCursor::getAttrs() auto & v = forceValue(); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector attrs; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 8c40c2565..e56ce261c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e14eb01c7..2f8d6d259 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -197,7 +197,7 @@ string showType(const Value & v) case tApp: return "a function application"; case tBlackhole: return "a black hole"; default: - return showType(v.normalType()); + return showType(v.type()); } } @@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.normalType() == nNull) + if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->normalType() != nAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->normalType() != nAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po return; } - if (fun.normalType() == nAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.normalType() == nAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1573,14 +1573,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.normalType(); + firstType = vTmp.type(); first = false; } if (firstType == nInt) { - if (vTmp.normalType() == nInt) { + if (vTmp.type() == nInt) { n += vTmp.integer; - } else if (vTmp.normalType() == nFloat) { + } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; nf = n; @@ -1588,9 +1588,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == nFloat) { - if (vTmp.normalType() == nInt) { + if (vTmp.type() == nInt) { nf += vTmp.integer; - } else if (vTmp.normalType() == nFloat) { + } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); @@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.normalType() == nAttrs) { + if (v.type() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nInt) + if (v.type() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() == nInt) + if (v.type() == nInt) return v.integer; - else if (v.normalType() != nFloat) + else if (v.type() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nFunction && !isFunctor(v)) + if (v.type() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nString) { + if (v.type() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.normalType() != nAttrs) return false; + if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->normalType() != nString) return false; + if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string s; - if (v.normalType() == nString) { + if (v.type() == nString) { copyContext(v, context); return v.string.s; } - if (v.normalType() == nPath) { + if (v.type() == nPath) { Path path(canonPath(v.path)); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.normalType() == nAttrs) { + if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.normalType() == nExternal) + if (v.type() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.normalType() == nBool && v.boolean) return "1"; - if (v.normalType() == nBool && !v.boolean) return ""; - if (v.normalType() == nInt) return std::to_string(v.integer); - if (v.normalType() == nFloat) return std::to_string(v.fpoint); - if (v.normalType() == nNull) return ""; + if (v.type() == nBool && v.boolean) return "1"; + if (v.type() == nBool && !v.boolean) return ""; + if (v.type() == nInt) return std::to_string(v.integer); + if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nNull) return ""; if (v.isList()) { string result; @@ -1884,15 +1884,15 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.normalType() == nInt && v2.normalType() == nFloat) + if (v1.type() == nInt && v2.type() == nFloat) return v1.integer == v2.fpoint; - if (v1.normalType() == nFloat && v2.normalType() == nInt) + if (v1.type() == nFloat && v2.type() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.normalType() != v2.normalType()) return false; + if (v1.type() != v2.type()) return false; - switch (v1.normalType()) { + switch (v1.type()) { case nInt: return v1.integer == v2.integer; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 987e7e24b..4f021570c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -82,9 +82,9 @@ static void expectType(EvalState & state, ValueType type, Value & value, const Pos & pos) { forceTrivialValue(state, value, pos); - if (value.normalType() != type) + if (value.type() != type) throw Error("expected %s but got %s at %s", - showType(type), showType(value.normalType()), pos); + showType(type), showType(value.type()), pos); } static std::map parseFlakeInputs( @@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->normalType() == nString) + if (attr.value->type() == nString) attrs.emplace(attr.name, attr.value->string.s); else throw TypeError("flake input attribute '%s' is %s while a string is expected", @@ -235,17 +235,17 @@ static Flake getFlake( for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); - if (setting.value->normalType() == nString) + if (setting.value->type() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nInt) + else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nBool) + else if (setting.value->type() == nBool) flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nList) { + else if (setting.value->type() == nList) { std::vector ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { auto elem = setting.value->listElems()[n]; - if (elem->normalType() != nString) + if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 93788273f..32c115c12 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->normalType() != nString) throw errMsg; + if ((*i)->type() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.normalType() == nList) { + if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString || - v.normalType() == nFloat; + else return v.type() == nInt || v.type() == nBool || v.type() == nString || + v.type() == nFloat; } @@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->normalType() != nString) return ""; + if (!v || v->type() != nString) return ""; return v->string.s; } @@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nInt) return v->integer; - if (v->normalType() == nString) { + if (v->type() == nInt) return v->integer; + if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ NixInt n; @@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nFloat) return v->fpoint; - if (v->normalType() == nString) { + if (v->type() == nFloat) return v->fpoint; + if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ NixFloat n; @@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nBool) return v->boolean; - if (v->normalType() == nString) { + if (v->type() == nBool) return v->boolean; + if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->normalType() == nAttrs) { + if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.normalType() == nList) { + else if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f6ca612f4..4106f1ec8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -356,7 +356,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->normalType()) { + switch (args[0]->type()) { case nInt: t = "int"; break; case nBool: t = "bool"; break; case nString: t = "string"; break; @@ -389,7 +389,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nNull); + mkBool(v, args[0]->type() == nNull); } static RegisterPrimOp primop_isNull({ @@ -409,7 +409,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nFunction); + mkBool(v, args[0]->type() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -425,7 +425,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nInt); + mkBool(v, args[0]->type() == nInt); } static RegisterPrimOp primop_isInt({ @@ -441,7 +441,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nFloat); + mkBool(v, args[0]->type() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -457,7 +457,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nString); + mkBool(v, args[0]->type() == nString); } static RegisterPrimOp primop_isString({ @@ -473,7 +473,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nBool); + mkBool(v, args[0]->type() == nBool); } static RegisterPrimOp primop_isBool({ @@ -489,7 +489,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nPath); + mkBool(v, args[0]->type() == nPath); } static RegisterPrimOp primop_isPath({ @@ -505,13 +505,13 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->normalType() == nFloat && v2->normalType() == nInt) + if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; - if (v1->normalType() == nInt && v2->normalType() == nFloat) + if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; - if (v1->normalType() != v2->normalType()) + if (v1->type() != v2->type()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->normalType()) { + switch (v1->type()) { case nInt: return v1->integer < v2->integer; case nFloat: @@ -762,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->normalType() == nString) + if (args[0]->type() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -887,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->normalType() == nNull) continue; + if (i->value->type() == nNull) continue; } if (i->name == state.sContentAddressed) { @@ -1293,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1793,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args }); state.forceValue(*args[0], pos); - if (args[0]->normalType() != nFunction) + if (args[0]->type() != nFunction) throw TypeError({ .hint = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", @@ -2059,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nAttrs); + mkBool(v, args[0]->type() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2322,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nList); + mkBool(v, args[0]->type() == nList); } static RegisterPrimOp primop_isList({ @@ -2816,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); @@ -2835,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); @@ -2854,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); @@ -2881,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) { + if (args[0]->type() == nFloat || args[1]->type() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2461ebc99..845a1ed1b 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6d93e1dc2..6e7ddde8e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,25 +85,25 @@ static void fetchTree( state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->normalType() == nPath || attr.value->normalType() == nString) + if (attr.value->type() == nPath || attr.value->type() == nString) addURI( state, attrs, attr.name, state.coerceToString(*attr.pos, *attr.value, context, false, false) ); - else if (attr.value->normalType() == nString) + else if (attr.value->type() == nString) addURI(state, attrs, attr.name, attr.value->string.s); - else if (attr.value->normalType() == nBool) + else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->normalType() == nInt) + else if (attr.value->type() == nInt) attrs.emplace(attr.name, attr.value->integer); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", @@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index b5f4c8654..bfea24d40 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,7 +16,7 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: out.write(v.integer); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 26be07cff..7464455d8 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -58,7 +58,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->normalType() == nString) + if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->normalType() == nString) + if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 8b312bf03..61ea1d64b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -136,12 +136,12 @@ public: // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type - // normalType() == nThunk + // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; inline bool isBlackhole() const { return internalType == tBlackhole; }; - // normalType() == nFunction + // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; @@ -204,7 +204,7 @@ public: // Returns the normal type of a Value. This only returns nThunk if the // Value hasn't been forceValue'd - inline ValueType normalType() const + inline ValueType type() const { switch (internalType) { case tInt: return nInt; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 404fd5111..6c2e075ed 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) i.queryName(), j) }); else { - if (v->normalType() == nString) { + if (v->type() == nString) { attrs2["type"] = "string"; attrs2["value"] = v->string.s; xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nInt) { + } else if (v->type() == nInt) { attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nFloat) { + } else if (v->type() == nFloat) { attrs2["type"] = "float"; attrs2["value"] = (format("%1%") % v->fpoint).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nBool) { + } else if (v->type() == nBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nList) { + } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->normalType() != nString) continue; + if (v->listElems()[j]->type() != nString) continue; XMLAttrs attrs3; attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } - } else if (v->normalType() == nAttrs) { + } else if (v->type() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); Bindings & attrs = *v->attrs; for (auto &i : attrs) { Attr & a(*attrs.find(i.name)); - if(a.value->normalType() != nString) continue; + if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = i.name; attrs3["value"] = a.value->string.s; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index bba3b1bc6..ea82e5300 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand recurse = [&](Value & v, const Pos & pos, const Path & path) { state->forceValue(v); - if (v.normalType() == nString) + if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path, v.string.s); - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e4da0348c..066430c5d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand if (v.isLambda()) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.normalType() == nAttrs) { + } else if (v.type() == nAttrs) { for (auto & attr : *v.attrs) try { state->forceValue(*attr.value, *attr.pos); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 047e2dc59..673155078 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -446,7 +446,7 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.normalType() == nPath || v.normalType() == nString) { + if (v.type() == nPath || v.type() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); @@ -669,7 +669,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; From 7de4b1e9aa72c21f5c98001ea2a5f312d26634b7 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Thu, 17 Dec 2020 23:42:49 +0100 Subject: [PATCH 213/477] smaller fixes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/command-ref/nix.md | 1 - doc/manual/src/contributing/hacking.md | 76 +++++++++++++++++++++++++ doc/manual/src/hacking.md | 77 -------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) delete mode 100644 doc/manual/src/command-ref/nix.md delete mode 100644 doc/manual/src/hacking.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index fdb1f7969..448fee803 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -62,6 +62,7 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) +@manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Glossary](glossary.md) diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md deleted file mode 100644 index cc677d127..000000000 --- a/doc/manual/src/command-ref/nix.md +++ /dev/null @@ -1 +0,0 @@ -# nix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2ad773dea..2a1e55e5b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -1 +1,77 @@ # Hacking + +This section provides some notes on how to hack on Nix. To get the +latest version of Nix from GitHub: + +```console +$ git clone https://github.com/NixOS/nix.git +$ cd nix +``` + +To build Nix for the current operating system/architecture use + +```console +$ nix-build +``` + +or if you have a flake-enabled nix: + +```console +$ nix build +``` + +This will build `defaultPackage` attribute defined in the `flake.nix` +file. To build for other platforms add one of the following suffixes to +it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. + +```console +$ nix-build -A defaultPackage.x86_64-linux +``` + +To build all dependencies and start a shell in which all environment +variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To build Nix itself in this shell: + +```console +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 3.0 +``` + +To run a functional test: + +```console +make tests/test-name-should-auto-complete.sh.test +``` + +To run the unit-tests for C++ code: + +``` +make check +``` + +If you have a flakes-enabled Nix you can replace: + +```console +$ nix-shell +``` + +by: + +```console +$ nix develop +``` diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md deleted file mode 100644 index 2a1e55e5b..000000000 --- a/doc/manual/src/hacking.md +++ /dev/null @@ -1,77 +0,0 @@ -# Hacking - -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -To build Nix for the current operating system/architecture use - -```console -$ nix-build -``` - -or if you have a flake-enabled nix: - -```console -$ nix build -``` - -This will build `defaultPackage` attribute defined in the `flake.nix` -file. To build for other platforms add one of the following suffixes to -it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. - -```console -$ nix-build -A defaultPackage.x86_64-linux -``` - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To build Nix itself in this shell: - -```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 3.0 -``` - -To run a functional test: - -```console -make tests/test-name-should-auto-complete.sh.test -``` - -To run the unit-tests for C++ code: - -``` -make check -``` - -If you have a flakes-enabled Nix you can replace: - -```console -$ nix-shell -``` - -by: - -```console -$ nix develop -``` From b70d22baca3e8826392b61aa53955c6da74b8724 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 18 Dec 2020 14:38:49 +0100 Subject: [PATCH 214/477] Replace Value type setters with mk* functions Move clearValue inside Value mkInt instead of setInt mkBool instead of setBool mkString instead of setString mkPath instead of setPath mkNull instead of setNull mkAttrs instead of setAttrs mkList instead of setList* mkThunk instead of setThunk mkApp instead of setApp mkLambda instead of setLambda mkBlackhole instead of setBlackhole mkPrimOp instead of setPrimOp mkPrimOpApp instead of setPrimOpApp mkExternal instead of setExternal mkFloat instead of setFloat Add note that the static mk* function should be removed eventually --- src/libexpr/attr-set.cc | 4 +- src/libexpr/eval-inline.hh | 6 +- src/libexpr/eval.cc | 49 +++------- src/libexpr/nixexpr.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/value.hh | 193 ++++++++++++++++++++++++------------- src/nix/repl.cc | 4 +- 7 files changed, 144 insertions(+), 116 deletions(-) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 17886a426..b6091c955 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) v = vEmptySet; return; } - clearValue(v); - v.setAttrs(); - v.attrs = allocBindings(capacity); + v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; } diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index e56ce261c..f6dead6b0 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -36,13 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos) Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.setBlackhole(); + v.mkBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.setThunk(); - v.thunk.env = env; - v.thunk.expr = expr; + v.mkThunk(env, expr); throw; } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2f8d6d259..5a641d02c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -430,9 +430,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) } } - clearValue(vEmptySet); - vEmptySet.setAttrs(); - vEmptySet.attrs = allocBindings(0); + vEmptySet.mkAttrs(allocBindings(0)); createBaseEnv(); } @@ -548,16 +546,14 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->setPrimOp(); - vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; + vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } Value * v = allocValue(); - v->setPrimOp(); - v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; + v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -572,8 +568,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->setPrimOp(); - vPrimOp->primOp = new PrimOp(std::move(primOp)); + vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(primOp.name, v); @@ -584,8 +579,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) primOp.name = symbols.create(std::string(primOp.name, 2)); Value * v = allocValue(); - v->setPrimOp(); - v->primOp = new PrimOp(std::move(primOp)); + v->mkPrimOp(new PrimOp(std::move(primOp))); staticBaseEnv.vars[envName] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); @@ -708,15 +702,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con void mkString(Value & v, const char * s) { - mkStringNoCopy(v, dupString(s)); + v.mkString(dupString(s)); } Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.setString(); - v.string.s = dupStringWithLen(s.data(), s.size()); - v.string.context = 0; + v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -731,7 +723,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) void mkPath(Value & v, const char * s) { - mkPathNoCopy(v, dupString(s)); + v.mkPath(dupString(s)); } @@ -792,16 +784,9 @@ Env & EvalState::allocEnv(size_t size) void EvalState::mkList(Value & v, size_t size) { - clearValue(v); - if (size == 1) - v.setList1(); - else if (size == 2) - v.setList2(); - else { - v.setListN(); - v.bigList.size = size; - v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; - } + v.mkList(size); + if (size > 2) + v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *)); nrListElems += size; } @@ -810,9 +795,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.setThunk(); - v.thunk.env = &env; - v.thunk.expr = expr; + v.mkThunk(&env, expr); nrThunks++; } @@ -1207,9 +1190,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.setLambda(); - v.lambda.env = &env; - v.lambda.fun = this; + v.mkLambda(&env, this); } @@ -1252,9 +1233,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.setPrimOpApp(); - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; + v.mkPrimOpApp(fun2, &arg); } } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..b80a7de4e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -129,7 +129,7 @@ struct ExprPath : Expr { string s; Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4106f1ec8..45066e9cf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1434,7 +1434,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - mkStringNoCopy(*ent_val, + ent_val->mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 61ea1d64b..b317c1898 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -114,24 +114,6 @@ friend void printValue(std::ostream & str, std::set & active, con public: - inline void setInt() { internalType = tInt; }; - inline void setBool() { internalType = tBool; }; - inline void setString() { internalType = tString; }; - inline void setPath() { internalType = tPath; }; - inline void setNull() { internalType = tNull; }; - inline void setAttrs() { internalType = tAttrs; }; - inline void setList1() { internalType = tList1; }; - inline void setList2() { internalType = tList2; }; - inline void setListN() { internalType = tListN; }; - inline void setThunk() { internalType = tThunk; }; - inline void setApp() { internalType = tApp; }; - inline void setLambda() { internalType = tLambda; }; - inline void setBlackhole() { internalType = tBlackhole; }; - inline void setPrimOp() { internalType = tPrimOp; }; - inline void setPrimOpApp() { internalType = tPrimOpApp; }; - inline void setExternal() { internalType = tExternal; }; - inline void setFloat() { internalType = tFloat; }; - // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type @@ -222,6 +204,123 @@ public: abort(); } + /* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ + inline void clearValue() + { + app.left = app.right = 0; + } + + inline void mkInt(NixInt n) + { + clearValue(); + internalType = tInt; + integer = n; + } + + inline void mkBool(bool b) + { + clearValue(); + internalType = tBool; + boolean = b; + } + + inline void mkString(const char * s, const char * * context = 0) + { + internalType = tString; + string.s = s; + string.context = context; + } + + inline void mkPath(const char * s) + { + clearValue(); + internalType = tPath; + path = s; + } + + inline void mkNull() + { + clearValue(); + internalType = tNull; + } + + inline void mkAttrs(Bindings * a) + { + clearValue(); + internalType = tAttrs; + attrs = a; + } + + inline void mkList(size_t size) + { + clearValue(); + if (size == 1) + internalType = tList1; + else if (size == 2) + internalType = tList2; + else { + internalType = tListN; + bigList.size = size; + } + } + + inline void mkThunk(Env * e, Expr * ex) + { + internalType = tThunk; + thunk.env = e; + thunk.expr = ex; + } + + inline void mkApp(Value * l, Value * r) + { + internalType = tApp; + app.left = l; + app.right = r; + } + + inline void mkLambda(Env * e, ExprLambda * f) + { + internalType = tLambda; + lambda.env = e; + lambda.fun = f; + } + + inline void mkBlackhole() + { + internalType = tBlackhole; + // Value will be overridden anyways + } + + inline void mkPrimOp(PrimOp * p) + { + clearValue(); + internalType = tPrimOp; + primOp = p; + } + + + inline void mkPrimOpApp(Value * l, Value * r) + { + internalType = tPrimOpApp; + app.left = l; + app.right = r; + } + + inline void mkExternal(ExternalValueBase * e) + { + clearValue(); + internalType = tExternal; + external = e; + } + + inline void mkFloat(NixFloat n) + { + clearValue(); + internalType = tFloat; + fpoint = n; + } + bool isList() const { return internalType == tList1 || internalType == tList2 || internalType == tListN; @@ -251,86 +350,42 @@ public: }; -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.left = v.app.right = 0; -} - +// TODO: Remove these static functions, replace call sites with v.mk* instead static inline void mkInt(Value & v, NixInt n) { - clearValue(v); - v.setInt(); - v.integer = n; + v.mkInt(n); } - static inline void mkFloat(Value & v, NixFloat n) { - clearValue(v); - v.setFloat(); - v.fpoint = n; + v.mkFloat(n); } - static inline void mkBool(Value & v, bool b) { - clearValue(v); - v.setBool(); - v.boolean = b; + v.mkBool(b); } - static inline void mkNull(Value & v) { - clearValue(v); - v.setNull(); + v.mkNull(); } - static inline void mkApp(Value & v, Value & left, Value & right) { - v.setApp(); - v.app.left = &left; - v.app.right = &right; + v.mkApp(&left, &right); } - -static inline void mkPrimOpApp(Value & v, Value & left, Value & right) -{ - v.setPrimOpApp(); - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.setString(); - v.string.s = s; - v.string.context = 0; -} - - static inline void mkString(Value & v, const Symbol & s) { - mkStringNoCopy(v, ((const string &) s).c_str()); + v.mkString(((const string &) s).c_str()); } void mkString(Value & v, const char * s); -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.setPath(); - v.path = s; -} - - void mkPath(Value & v, const char * s); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 673155078..a992d8732 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -551,9 +551,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.setThunk(); - v.thunk.env = env; - v.thunk.expr = e; + v.mkThunk(env, e); addVarToScope(state->symbols.create(name), v); } else { Value v; From 1a1af75338cb9ed28dc00de2e696d8efc5d37287 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 15:33:12 +0000 Subject: [PATCH 215/477] Overhaul store subclassing We embrace virtual the rest of the way, and get rid of the `assert(false)` 0-param constructors. We also list config base classes first, so the constructor order is always: 1. all the configs 2. all the stores Each in the same order --- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build/derivation-goal.cc | 11 ++++++++--- src/libstore/dummy-store.cc | 3 ++- src/libstore/http-binary-cache-store.cc | 5 ++++- src/libstore/legacy-ssh-store.cc | 3 ++- src/libstore/local-binary-cache-store.cc | 5 ++++- src/libstore/local-fs-store.hh | 2 +- src/libstore/local-store.cc | 2 ++ src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 11 ++++++++++- src/libstore/s3-binary-cache-store.hh | 6 ++---- src/libstore/ssh-store.cc | 4 +++- src/libstore/store-api.hh | 20 +------------------- src/libstore/uds-remote-store.cc | 3 +++ src/libstore/uds-remote-store.hh | 7 +------ 17 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 07a8b2beb..443a53cac 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "enable multi-threading compression, available for xz only currently"}; }; -class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store { private: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f494545fb..47d11dc53 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1985,7 +1985,7 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } -struct RestrictedStoreConfig : LocalFSStoreConfig +struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; const std::string name() { return "Restricted Store"; } @@ -1994,14 +1994,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore { ref next; DerivationGoal & goal; RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) { } Path getRealStoreDir() override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 91fc178db..3c7caf8f2 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig { const std::string name() override { return "Dummy Store"; } }; -struct DummyStore : public Store, public virtual DummyStoreConfig +struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(const std::string scheme, const std::string uri, const Params & params) : DummyStore(params) @@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) + , DummyStoreConfig(params) , Store(params) { } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d2a89f96..0a3afcd51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Http Binary Cache Store"; } }; -class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig +class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -36,6 +36,9 @@ public: const Path & _cacheUri, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , cacheUri(scheme + "://" + _cacheUri) { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ad1779aea..253c0033e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig const std::string name() override { return "Legacy SSH Store"; } }; -struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in @@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig LegacySSHStore(const string & scheme, const string & host, const Params & params) : StoreConfig(params) + , LegacySSHStoreConfig(params) , Store(params) , host(host) , connections(make_ref>( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index bb7464989..a58b7733f 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Local Binary Cache Store"; } }; -class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig +class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -24,6 +24,9 @@ public: const Path & binaryCacheDir, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8eccd8236..55941b771 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "log", "directory where Nix will store state"}; }; -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store { public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9f9bde4d..c52d4b62a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -100,6 +100,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) LocalStore::LocalStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 877dba742..ae9497b2e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public LocalFSStore, public virtual LocalStoreConfig +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore { private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f1f4d0516..be07f02dc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) + : RemoteStoreConfig(params) + , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fdd53e6ed..b3a9910a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store { public: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d6edafd7e..6bfbee044 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } +S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) + : BinaryCacheStoreConfig(params) + , BinaryCacheStore(params) +{ } + struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; @@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "S3 Binary Cache Store"; } }; -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { std::string bucketName; @@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache const std::string & bucketName, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(params) + , Store(params) + , BinaryCacheStore(params) , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..bce828b11 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -6,13 +6,11 @@ namespace nix { -class S3BinaryCacheStore : public BinaryCacheStore +class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: - S3BinaryCacheStore(const Params & params) - : BinaryCacheStore(params) - { } + S3BinaryCacheStore(const Params & params); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 08d0bd565..17c258201 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig const std::string name() override { return "SSH Store"; } }; -class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig +class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , RemoteStoreConfig(params) + , SSHStoreConfig(params) , Store(params) , RemoteStore(params) , host(host) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ce95b78b1..9bcff08eb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -175,25 +175,7 @@ struct StoreConfig : public Config { using Config::Config; - /** - * When constructing a store implementation, we pass in a map `params` of - * parameters that's supposed to initialize the associated config. - * To do that, we must use the `StoreConfig(StringMap & params)` - * constructor, so we'd like to `delete` its default constructor to enforce - * it. - * - * However, actually deleting it means that all the subclasses of - * `StoreConfig` will have their default constructor deleted (because it's - * supposed to call the deleted default constructor of `StoreConfig`). But - * because we're always using virtual inheritance, the constructors of - * child classes will never implicitely call this one, so deleting it will - * be more painful than anything else. - * - * So we `assert(false)` here to ensure at runtime that the right - * constructor is always called without having to redefine a custom - * constructor for each `*Config` class. - */ - StoreConfig() { assert(false); } + StoreConfig() = delete; virtual ~StoreConfig() { } diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 24f3e9c6d..cac4fa036 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -15,6 +15,9 @@ namespace nix { UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , UDSRemoteStoreConfig(params) , Store(params) , LocalFSStore(params) , RemoteStore(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index e5de104c9..ddc7716cd 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon { } - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - const std::string name() override { return "Local Daemon Store"; } }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore { public: From 346baec783a7423aa5b6cacaf7eebb8d22d4ce79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Dec 2020 13:04:24 +0100 Subject: [PATCH 216/477] Move doc() to Args --- src/libutil/args.cc | 4 ++-- src/libutil/args.hh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 61f9503ec..a929ea5ac 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -254,6 +254,8 @@ nlohmann::json Args::toJSON() res["description"] = description(); res["flags"] = std::move(flags); res["args"] = std::move(args); + auto s = doc(); + if (s != "") res.emplace("doc", stripIndentation(s)); return res; } @@ -378,8 +380,6 @@ nlohmann::json Command::toJSON() auto res = Args::toJSON(); res["examples"] = std::move(exs); - auto s = doc(); - if (s != "") res.emplace("doc", stripIndentation(s)); return res; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 8069fd70f..68bbbb4f7 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -25,6 +25,9 @@ public: /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } + /* Return documentation about this command, in Markdown format. */ + virtual std::string doc() { return ""; } + protected: static const size_t ArityAny = std::numeric_limits::max(); @@ -225,9 +228,6 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; - /* Return documentation about this command, in Markdown format. */ - virtual std::string doc() { return ""; } - struct Example { std::string description; From ae7351dbeed3d03087b6b56d23b4b3942de1507d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 14:19:36 +0100 Subject: [PATCH 217/477] Add 'nix build' manpage --- src/nix/build.cc | 19 +++------- src/nix/build.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 src/nix/build.md diff --git a/src/nix/build.cc b/src/nix/build.cc index 67be4024b..c2974d983 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -43,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile return "build a derivation or fetch a store path"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To build and run GNU Hello from NixOS 17.03:", - "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" - }, - Example{ - "To build the build.x86_64-linux attribute from release.nix:", - "nix build -f release.nix build.x86_64-linux" - }, - Example{ - "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs#hello" - }, - }; + return + #include "build.md" + ; } void run(ref store) override diff --git a/src/nix/build.md b/src/nix/build.md new file mode 100644 index 000000000..c2f3e387a --- /dev/null +++ b/src/nix/build.md @@ -0,0 +1,92 @@ +R""( + +# Examples + +* Build the default package from the flake in the current directory: + + ```console + # nix build + ``` + +* Build and run GNU Hello from the `nixpkgs` flake: + + ```console + # nix build nixpkgs#hello + # ./result/bin/hello + Hello, world! + ``` + +* Build GNU Hello and Cowsay, leaving two result symlinks: + + ```console + # nix build nixpkgs#hello nixpkgs#cowsay + # ls -l result* + lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +* Build a specific output: + + ```console + # nix build nixpkgs#glibc.dev + # ls -ld ./result-dev + lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev + ``` + +* Build attribute `build.x86_64-linux` from (non-flake) Nix expression + `release.nix`: + + ```console + # nix build -f release.nix build.x86_64-linux + ``` + +* Build a NixOS system configuration from a flake, and make a profile + point to the result: + + ```console + # nix build --profile /nix/var/nix/profiles/system \ + ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel + ``` + + (This is essentially what `nixos-rebuild` does.) + +* Build an expression specified on the command line: + + ```console + # nix build --impure --expr \ + 'with import {}; + runCommand "foo" { + buildInputs = [ hello ]; + } + "hello > $out"' + # cat ./result + Hello, world! + ``` + + Note that `--impure` is needed because we're using ``, + which relies on the `$NIX_PATH` environment variable. + +* Fetch a store path from the configured substituters, if it doesn't + already exist: + + ```console + # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +# Description + +`nix build` builds the specified *installables*. Installables that +resolve to derivations are built (or substituted if possible). Store +path installables are substituted. + +Unless `--no-link` is specified, after a successful build, it creates +symlinks to the store paths of the installables. These symlinks have +the prefix `./result` by default; this can be overriden using the +`--out-link` option. Each symlink has a suffix `--`, where +*N* is the index of the installable (with the left-most installable +having index 0), and *outname* is the symbolic derivation output name +(e.g. `bin`, `dev` or `lib`). `-` is omitted if *N* = 0, and +`-` is omitted if *outname* = `out` (denoting the default +output). + +)"" From 09660b855778531be14968b720308d092af4dd2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:16:23 +0100 Subject: [PATCH 218/477] Add 'nix run' and 'nix shell' manpages --- src/nix/run.cc | 42 +++++------------------ src/nix/run.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ src/nix/shell.md | 48 ++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 34 deletions(-) create mode 100644 src/nix/run.md create mode 100644 src/nix/shell.md diff --git a/src/nix/run.cc b/src/nix/run.cc index 92a52c6cd..c2400f9ee 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -86,26 +86,11 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment return "run a shell in which the specified packages are available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To start a shell providing GNU Hello from NixOS 20.03:", - "nix shell nixpkgs/nixos-20.03#hello" - }, - Example{ - "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs#youtube-dl" - }, - Example{ - "To run GNU Hello:", - "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'" - }, - Example{ - "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs#hello -c hello" - }, - }; + return + #include "shell.md" + ; } void run(ref store) override @@ -168,22 +153,11 @@ struct CmdRun : InstallableCommand, RunCommon return "run a Nix application"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To run Blender:", - "nix run blender-bin" - }, - Example{ - "To run vim from nixpkgs:", - "nix run nixpkgs#vim" - }, - Example{ - "To run vim from nixpkgs with arguments:", - "nix run nixpkgs#vim -- --help" - }, - }; + return + #include "run.md" + ; } Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/run.md b/src/nix/run.md new file mode 100644 index 000000000..c178e8b13 --- /dev/null +++ b/src/nix/run.md @@ -0,0 +1,87 @@ +R""( + +# Examples + +* Run the default app from the `blender-bin` flake: + + ```console + # nix run blender-bin + ``` + +* Run a non-default app from the `blender-bin` flake: + + ```console + # nix run blender-bin#blender_2_83 + ``` + + Tip: you can find apps provided by this flake by running `nix flake + show blender-bin`. + +* Run `vim` from the `nixpkgs` flake: + + ```console + # nix run nixpkgs#vim + ``` + + Note that `vim` (as of the time of writing of this page) is not an + app but a package. Thus, Nix runs the eponymous file from the `vim` + package. + +* Run `vim` with arguments: + + ```console + # nix run nixpkgs#vim -- --help + ``` + +# Description + +`nix run` builds and runs *installable*, which must evaluate to an +*app* or a regular Nix derivation. + +If *installable* evaluates to an *app* (see below), it executes the +program specified by the app definition. + +If *installable* evaluates to a derivation, it will try to execute the +program `/bin/`, where *out* is the primary output store +path of the derivation and *name* is the name part of the value of the +`name` attribute of the derivation (e.g. if `name` is set to +`hello-1.10`, it will run `$out/bin/hello`). + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `defaultApp.` + +* `defaultPackage.` + +If an attribute *name* is given, `nix run` tries the following flake +output attributes: + +* `apps..` + +* `packages..` + +* `legacyPackages..` + +# Apps + +An app is specified by a flake output attribute named +`apps..` or `defaultApp.`. It looks like this: + +```nix +apps.x86_64-linux.blender_2_79 = { + type = "app"; + program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; +}; +``` + +The only supported attributes are: + +* `type` (required): Must be set to `app`. + +* `program` (required): The full path of the executable to run. It + must reside in the Nix store. + +)"" diff --git a/src/nix/shell.md b/src/nix/shell.md new file mode 100644 index 000000000..2a379e03f --- /dev/null +++ b/src/nix/shell.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Start a shell providing `youtube-dl` from the `nixpkgs` flake: + + ```console + # nix shell nixpkgs#youtube-dl + # youtube-dl --version + 2020.11.01.1 + ``` + +* Start a shell providing GNU Hello from NixOS 20.03: + + ```console + # nix shell nixpkgs/nixos-20.03#hello + ``` + +* Run GNU Hello: + + ```console + # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + Hi everybody! + ``` + +* Run GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello -c hello + ``` + +* Start a shell providing GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + ``` + + Note that it's necessary to specify `bash` explicitly because your + default shell (e.g. `/bin/bash`) generally will not exist in the + chroot. + +# Description + +`nix shell` runs a command in an environment in which the `$PATH` +variable provides the specified *installables*. If not command is +specified, it starts the default shell of your user account. + +)"" From 9dcd0aebc59a53c622d709c33d4c6f5e20bc0ac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:34:45 +0100 Subject: [PATCH 219/477] generate-manpage.nix: Fix short names --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fbd7f3e7d..9f30c8fbc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -43,7 +43,7 @@ let if flag.category or "" != "config" then " - `--${longName}`" - + (if flag ? shortName then " / `${flag.shortName}`" else "") + + (if flag ? shortName then " / `-${flag.shortName}`" else "") + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") + " \n" + " " + flag.description + "\n\n" From 28ee307fd8d38e8f6c8dd9d2575435036f3612cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:49:58 +0100 Subject: [PATCH 220/477] Add 'nix copy' manpage --- src/nix/copy.cc | 29 ++++--------------------- src/nix/copy.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 src/nix/copy.md diff --git a/src/nix/copy.cc b/src/nix/copy.cc index cb31aac8f..2394eb46d 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -54,32 +54,11 @@ struct CmdCopy : StorePathsCommand return "copy paths between Nix stores"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache $(type -p firefox)" - }, - Example{ - "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server /run/current-system" - }, - Example{ - "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" - }, -#ifdef ENABLE_S3 - Example{ - "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello" - }, - Example{ - "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello" - }, -#endif - }; + return + #include "copy.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/copy.md b/src/nix/copy.md new file mode 100644 index 000000000..25e0ddadc --- /dev/null +++ b/src/nix/copy.md @@ -0,0 +1,58 @@ +R""( + +# Examples + +* Copy Firefox from the local store to a binary cache in `/tmp/cache`: + + ```console + # nix copy --to file:///tmp/cache $(type -p firefox) + ``` + + Note the `file://` - without this, the destination is a chroot + store, not a binary cache. + +* Copy the entire current NixOS system closure to another machine via + SSH: + + ```console + # nix copy -s --to ssh://server /run/current-system + ``` + + The `-s` flag causes the remote machine to try to substitute missing + store paths, which may be faster if the link between the local and + remote machines is slower than the link between the remote machine + and its substituters (e.g. `https://cache.nixos.org`). + +* Copy a closure from another machine via SSH: + + ```console + # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2 + ``` + +* Copy Hello to a binary cache in an Amazon S3 bucket: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello + ``` + + or to an S3-compatible storage system: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello + ``` + + Note that this only works if Nix is built with AWS support. + +* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`: + + ```console + # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs + ``` + +# Description + +`nix copy` copies store path closures between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)"" From e9de689a6efca961099f1acfc574009f31a6f130 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 18:09:30 +0100 Subject: [PATCH 221/477] Add 'nix search' manpage --- src/nix/search.cc | 25 ++++------------ src/nix/search.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 20 deletions(-) create mode 100644 src/nix/search.md diff --git a/src/nix/search.cc b/src/nix/search.cc index 47770e128..9f864b3a4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -41,29 +41,14 @@ struct CmdSearch : InstallableCommand, MixJSON std::string description() override { - return "query available packages"; + return "search for packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show all packages in the flake in the current directory:", - "nix search" - }, - Example{ - "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:", - "nix search nixpkgs blender" - }, - Example{ - "To search for Firefox or Chromium:", - "nix search nixpkgs 'firefox|chromium'" - }, - Example{ - "To search for packages containing 'git' and either 'frontend' or 'gui':", - "nix search nixpkgs git 'frontend|gui'" - } - }; + return + #include "search.md" + ; } Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/search.md b/src/nix/search.md new file mode 100644 index 000000000..d182788a6 --- /dev/null +++ b/src/nix/search.md @@ -0,0 +1,72 @@ +R""( + +# Examples + +* Show all packages in the `nixpkgs` flake: + + ```console + # nix search nixpkgs + * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) + A set of ambisonics ladspa plugins + + * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0) + Application with primary purpose of idling Steam cards from multiple accounts simultaneously + … + ``` + +* Show packages in the `nixpkgs` flake containing `blender` in its + name or description: + + ```console + # nix search nixpkgs blender + * legacyPackages.x86_64-linux.blender (2.91.0) + 3D Creation/Animation/Publishing System + ``` + +* Search for packages underneath the attribute `gnome3` in Nixpkgs: + + ```console + # nix search nixpkgs#gnome3 vala + * legacyPackages.x86_64-linux.gnome3.vala (0.48.9) + Compiler for GObject type system + ``` + +* Show all packages in the flake in the current directory: + + ```console + # nix search + ``` + +* Search for Firefox or Chromium: + + ```console + # nix search nixpkgs 'firefox|chromium' + ``` + +* Search for packages containing `git'`and either `frontend` or `gui`: + + ```console + # nix search nixpkgs git 'frontend|gui' + ``` + +# Description + +`nix search` searches *installable* (which must be evaluatable, e.g. a +flake) for packages whose name or description matches all of the +regular expressions *regex*. For each matching package, It prints the +full attribute name (from the root of the installable), the version +and the `meta.description` field, highlighting the substrings that +were matched by the regular expressions. If no regular expressions are +specified, all packages are shown. + +# Flake output attributes + +If no flake output attribute is given, `nix search` searches for +packages: + +* Directly underneath `packages.`. + +* Underneath `legacyPackages.`, recursing into attribute sets + that contain an attribute `recurseForDerivations = true`. + +)"" From 42cc98f8d66627ff7e396fb809034d3389b3bd0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 21:52:22 +0100 Subject: [PATCH 222/477] Add 'nix develop' and `nix print-dev-env' manpages --- src/nix/develop.cc | 38 ++++------------ src/nix/develop.md | 94 ++++++++++++++++++++++++++++++++++++++++ src/nix/print-dev-env.md | 19 ++++++++ 3 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 src/nix/develop.md create mode 100644 src/nix/print-dev-env.md diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 457d94382..edd87f246 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -385,30 +385,11 @@ struct CmdDevelop : Common, MixEnvironment return "run a bash shell that provides the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build environment of GNU hello:", - "nix develop nixpkgs#hello" - }, - Example{ - "To get the build environment of the default package of flake in the current directory:", - "nix develop" - }, - Example{ - "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs#hello" - }, - Example{ - "To use a build environment previously recorded in a profile:", - "nix develop /tmp/my-shell" - }, - Example{ - "To replace all occurences of a store path with a writable directory:", - "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev" - }, - }; + return + #include "develop.md" + ; } void run(ref store) override @@ -495,14 +476,11 @@ struct CmdPrintDevEnv : Common return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs#hello)" - }, - }; + return + #include "print-dev-env.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/develop.md b/src/nix/develop.md new file mode 100644 index 000000000..7a906d10d --- /dev/null +++ b/src/nix/develop.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Start a shell with the build environment of the default package of + the flake in the current directory: + + ```console + # nix develop + ``` + + Typical commands to run inside this shell are: + + ```console + # configurePhase + # buildPhase + # installPhase + ``` + + Alternatively, you can run whatever build tools your project uses + directly, e.g. for a typical Unix project: + + ```console + # ./configure --prefix=$out + # make + # make install + ``` + +* Run a particular build phase directly: + + ```console + # nix develop --configure + # nix develop --build + # nix develop --check + # nix develop --install + # nix develop --installcheck + ``` + +* Start a shell with the build environment of GNU Hello: + + ```console + # nix develop nixpkgs#hello + ``` + +* Record a build environment in a profile: + + ```console + # nix develop --profile /tmp/my-build-env nixpkgs#hello + ``` + +* Use a build environment previously recorded in a profile: + + ```cosnole + # nix develop /tmp/my-build-env + ``` + +* Replace all occurences of the store path corresponding to + `glibc.dev` with a writable directory: + + ```console + # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev + ``` + + Note that this is useful if you're running a `nix develop` shell for + `nixpkgs#glibc` in `~/my-glibc` and want to compile another package + against it. + +# Description + +`nix develop` starts a `bash` shell that provides an interactive build +environment nearly identical to what Nix would use to build +*installable*. Inside this shell, environment variables and shell +functions are set up so that you can interactively and incrementally +build your package. + +Nix determines the build environment by building a modified version of +the derivation *installable* that just records the environment +initialised by `stdenv` and exits. This build environment can be +recorded into a profile using `--profile`. + +The prompt used by the `bash` shell can be customised by setting the +`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in +the flake's `nixConfig` attribute. + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `devShell.` + +* `defaultPackage.` + +)"" diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md new file mode 100644 index 000000000..b80252acf --- /dev/null +++ b/src/nix/print-dev-env.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Apply the build environment of GNU hello to the current shell: + + ```console + # . <(nix print-dev-env nixpkgs#hello) + ``` + +# Description + +This command prints a shell script that can be sourced by `b`ash and +that sets the environment variables and shell functions defined by the +build process of *installable*. This allows you to get a similar build +environment in your current shell rather than in a subshell (as with +`nix develop`). + +)"" From e90e74523238f37748d9f406732919374d7ee561 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 22:57:14 +0100 Subject: [PATCH 223/477] Add 'nix registry' manpages This also documents the registry format and matching/unification semantics (though not quite correctly). --- src/nix/registry-add.md | 33 +++++++++++++ src/nix/registry-list.md | 29 +++++++++++ src/nix/registry-pin.md | 38 +++++++++++++++ src/nix/registry-remove.md | 16 +++++++ src/nix/registry.cc | 35 ++++++++++++++ src/nix/registry.md | 98 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+) create mode 100644 src/nix/registry-add.md create mode 100644 src/nix/registry-list.md create mode 100644 src/nix/registry-pin.md create mode 100644 src/nix/registry-remove.md create mode 100644 src/nix/registry.md diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md new file mode 100644 index 000000000..80a31996a --- /dev/null +++ b/src/nix/registry-add.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03 + ``` + +* Pin `nixpkgs` to a specific revision: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + +* Add an entry that redirects a specific branch of `nixpkgs` to + another fork: + + ```console + # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *from-url* to flake reference *to-url*. If an entry for +*from-url* already exists, it is overwritten. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md new file mode 100644 index 000000000..30b6e29d8 --- /dev/null +++ b/src/nix/registry-list.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Show the contents of all registries: + + ```console + # nix registry list + user flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5 + system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124 + global flake:blender-bin github:edolstra/nix-warez?dir=blender + global flake:dwarffs github:edolstra/dwarffs + … + ``` + +# Description + +This command displays the contents of all registries on standard +output. Each line represents one registry entry in the format *type* +*from* *to*, where *type* denotes the registry containing the entry: + +* `flags`: entries specified on the command line using `--override-flake`. +* `user`: the user registry. +* `system`: the system registry. +* `global`: the global registry. + +See the [`nix registry` manual page](./nix3-registry.md) for more details. + +)"" diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md new file mode 100644 index 000000000..6e97e003e --- /dev/null +++ b/src/nix/registry-pin.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Pin `nixpkgs` to its most recent Git revision: + + ```console + # nix registry pin nixpkgs + ``` + + Afterwards the user registry will have an entry like this: + + ```console + nix registry list | grep '^user ' + user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + + and `nix flake info` will say: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + … + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *url* to the corresponding *locked* flake reference, that +is, a flake reference that specifies an exact revision or content +hash. This ensures that until this registry entry is removed, all uses +of *url* will resolve to exactly the same flake. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md new file mode 100644 index 000000000..4c0eb4947 --- /dev/null +++ b/src/nix/registry-remove.md @@ -0,0 +1,16 @@ +R""( + +# Examples + +* Remove the entry `nixpkgs` from the user registry: + + ```console + # nix registry remove nixpkgs + ``` + +# Description + +This command removes from the user registry any entry for flake +reference *url*. + +)"" diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 9352e00a7..f9719600f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -17,6 +17,13 @@ struct CmdRegistryList : StoreCommand return "list available Nix flakes"; } + std::string doc() override + { + return + #include "registry-list.md" + ; + } + void run(nix::ref store) override { using namespace fetchers; @@ -47,6 +54,13 @@ struct CmdRegistryAdd : MixEvalArgs, Command return "add/replace flake in user flake registry"; } + std::string doc() override + { + return + #include "registry-add.md" + ; + } + CmdRegistryAdd() { expectArg("from-url", &fromUrl); @@ -75,6 +89,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command return "remove flake from user flake registry"; } + std::string doc() override + { + return + #include "registry-remove.md" + ; + } + CmdRegistryRemove() { expectArg("url", &url); @@ -97,6 +118,13 @@ struct CmdRegistryPin : virtual Args, EvalCommand return "pin a flake to its current version in user flake registry"; } + std::string doc() override + { + return + #include "registry-pin.md" + ; + } + CmdRegistryPin() { expectArg("url", &url); @@ -132,6 +160,13 @@ struct CmdRegistry : virtual NixMultiCommand return "manage the flake registry"; } + std::string doc() override + { + return + #include "registry.md" + ; + } + Category category() override { return catSecondary; } void run() override diff --git a/src/nix/registry.md b/src/nix/registry.md new file mode 100644 index 000000000..557e5795b --- /dev/null +++ b/src/nix/registry.md @@ -0,0 +1,98 @@ +R""( + +# Description + +`nix flake` provides subcommands for managing *flake +registries*. Flake registries are a convenience feature that allows +you to refer to flakes using symbolic identifiers such as `nixpkgs`, +rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You +can use these identifiers on the command line (e.g. when you do `nix +run nixpkgs#hello`) or in flake input specifications in `flake.nix` +files. The latter are automatically resolved to full URLs and recorded +in the flake's `flake.lock` file. + +In addition, the flake registry allows you to redirect arbitrary flake +references (e.g. `github:NixOS/patchelf`) to another location, such as +a local fork. + +There are multiple registries. These are, in order from lowest to +highest precedence: + +* The global registry, which is a file downloaded from the URL + specified by the setting `flake-registry`. It is cached locally and + updated automatically when it's older than `tarball-ttl` + seconds. The default global registry is kept in [a GitHub + repository](https://github.com/NixOS/flake-registry). + +* The system registry, which is shared by all users. The default + location is `/etc/nix/registry.json`. On NixOS, the system registry + can be specified using the NixOS option `nix.registry`. + +* The user registry `~/.config/nix/registry.json`. This registry can + be modified by commands such as `nix flake pin`. + +* Overrides specified on the command line using the option + `--override-flake`. + +# Registry format + +A registry is a JSON file with the following format: + +```json +{ + "version": 2, + [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "github", + "owner": "NixOS", + "repo": "nixpkgs" + } + }, + ... + ] +} +``` + +That is, it contains a list of objects with attributes `from` and +`to`, both of which contain a flake reference in attribute +representation. (For example, `{"type": "indirect", "id": "nixpkgs"}` +is the attribute representation of `nixpkgs`, while `{"type": +"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute +representation of `github:NixOS/nixpkgs`.) + +Given some flake reference *R*, a registry entry is used if its +`from` flake reference *matches* *R*. *R* is then replaced by the +*unification* of the `to` flake reference with *R*. + +# Matching + +The `from` flake reference in a registry entry *matches* some flake +reference *R* if the attributes in `from` are the same as the +attributes in `R`. For example: + +* `nixpkgs` matches with `nixpkgs`. + +* `nixpkgs` matches with `nixpkgs/nixos-20.09`. + +* `nixpkgs/nixos-20.09` does not match with `nixpkgs`. + +* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`. + +# Unification + +The `to` flake reference in a registry entry is *unified* with some flake +reference *R* by taking `to` and applying the `rev` and `ref` +attributes from *R*, if specified. For example: + +* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`. + +* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +)"" From 4e065229c75455d3cee8bc9e791c10faa93c1b50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 23:01:50 +0100 Subject: [PATCH 224/477] Typo --- src/nix/develop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.md b/src/nix/develop.md index 7a906d10d..e71d9f8aa 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -50,7 +50,7 @@ R""( * Use a build environment previously recorded in a profile: - ```cosnole + ```console # nix develop /tmp/my-build-env ``` From b2262be19babc37a54bed4384fecba11d87f7364 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 12:55:24 +0100 Subject: [PATCH 225/477] Add 'nix edit' manpage --- src/nix/edit.cc | 11 ++++------- src/nix/edit.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 src/nix/edit.md diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 51c16f5a9..6472dd27a 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand return "open the Nix expression of a Nix package in $EDITOR"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs#hello" - }, - }; + return + #include "edit.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/edit.md b/src/nix/edit.md new file mode 100644 index 000000000..80563d06b --- /dev/null +++ b/src/nix/edit.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Open the Nix expression of the GNU Hello package: + + ```console + # nix edit nixpkgs#hello + ``` + +* Get the filename and line number used by `nix edit`: + + ```console + # nix eval --raw nixpkgs#hello.meta.position + /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15 + ``` + +# Description + +This command opens the Nix expression of a derivation in an +editor. The filename and line number of the derivation are taken from +its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets +this attribute to the location of the definition of the +`meta.description`, `version` or `name` derivation attributes. + +The editor to invoke is specified by the `EDITOR` environment +variable. It defaults to `cat`. If the editor is `emacs`, `nano` or +`vim`, it is passed the line number of the derivation using the +argument `+`. + +)"" From 6ce393392b4e303f51a512d64627cbd68a73c0e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 13:07:01 +0100 Subject: [PATCH 226/477] Add 'nix repl' manpage --- src/nix/repl.cc | 12 +++++------ src/nix/repl.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 src/nix/repl.md diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a992d8732..bce8d31dc 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -405,6 +405,7 @@ bool NixRepl::processLine(string line) } if (command == ":?" || command == ":help") { + // FIXME: convert to Markdown, include in the 'nix repl' manpage. std::cout << "The following commands are available:\n" << "\n" @@ -801,14 +802,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "Display all special commands within the REPL:", - "nix repl\nnix-repl> :?" - } - }; + return + #include "repl.md" + ; } void run(ref store) override diff --git a/src/nix/repl.md b/src/nix/repl.md new file mode 100644 index 000000000..bba60f871 --- /dev/null +++ b/src/nix/repl.md @@ -0,0 +1,57 @@ +R""( + +# Examples + +* Display all special commands within the REPL: + + ```console + # nix repl + nix-repl> :? + ``` + +* Evaluate some simple Nix expressions: + + ```console + # nix repl + + nix-repl> 1 + 2 + 3 + + nix-repl> map (x: x * 2) [1 2 3] + [ 2 4 6 ] + ``` + +* Interact with Nixpkgs in the REPL: + + ```console + # nix repl '' + + Loading ''... + Added 12428 variables. + + nix-repl> emacs.name + "emacs-27.1" + + nix-repl> emacs.drvPath + "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" + + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + + nix-repl> :b x + this derivation produced the following outputs: + out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello + + nix-repl> builtins.readFile drv + "Hello, world!\n" + ``` + +# Description + +This command provides an interactive environment for evaluating Nix +expressions. (REPL stands for 'read–eval–print loop'.) + +On startup, it loads the Nix expressions named *files* and adds them +into the lexical scope. You can load addition files using the `:l +` command, or reload all files using `:r`. + +)"" From 58bacc85e79b318f28e456a72be2d6a7c8d86991 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 13:41:47 +0100 Subject: [PATCH 227/477] Add 'nix log' manpage --- src/nix/log.cc | 19 ++++--------------- src/nix/log.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 src/nix/log.md diff --git a/src/nix/log.cc b/src/nix/log.cc index 33a3053f5..67d3742d6 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand return "show the build log of the specified packages or paths, if available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build log of GNU Hello:", - "nix log nixpkgs#hello" - }, - Example{ - "To get the build log of a specific path:", - "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" - }, - Example{ - "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs#hello" - }, - }; + return + #include "log.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/log.md b/src/nix/log.md new file mode 100644 index 000000000..8ee2f1d19 --- /dev/null +++ b/src/nix/log.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Get the build log of GNU Hello: + + ```console + # nix log nixpkgs#hello + ``` + +* Get the build log of a specific store path: + + ```console + # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1 + ``` + +* Get a build log from a specific binary cache: + + ```console + # nix log --store https://cache.nixos.org nixpkgs#hello + ``` + +# Description + +This command prints the log of a previous build of a derivation on +standard output. + +Nix looks for build logs in two places: + +* In the directory `/nix/var/log/nix/drvs`, which contains logs for + locally built derivations. + +* In the binary caches listed in the `substituters` setting. Logs + should be named `/log/`, where + `store-path` is a derivation, + e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`. + For non-derivation store paths, Nix will first try to determine the + deriver by fetching the `.narinfo` file for this store path. + +)"" From f34b1801a4ec9060504a6b712b52763e88b4341f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 17:41:58 +0100 Subject: [PATCH 228/477] Tweak --- src/nix/log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/log.md b/src/nix/log.md index 8ee2f1d19..1c76226a3 100644 --- a/src/nix/log.md +++ b/src/nix/log.md @@ -22,8 +22,8 @@ R""( # Description -This command prints the log of a previous build of a derivation on -standard output. +This command prints the log of a previous build of the derivation +*installable* on standard output. Nix looks for build logs in two places: From 53ce20eab779e0dd156af2ac9ec2318814edc7cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 17:55:59 +0100 Subject: [PATCH 229/477] Add 'nix store ping' manpage --- src/nix/ping-store.cc | 13 +++++-------- src/nix/ping-store.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 src/nix/ping-store.md diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 19b1a55c8..62b645b06 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -8,17 +8,14 @@ struct CmdPingStore : StoreCommand { std::string description() override { - return "test whether a store can be opened"; + return "test whether a store can be accessed"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To test whether connecting to a remote Nix store via SSH works:", - "nix store ping --store ssh://mac1" - }, - }; + return + #include "ping-store.md" + ; } void run(ref store) override diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md new file mode 100644 index 000000000..322093091 --- /dev/null +++ b/src/nix/ping-store.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* Test whether connecting to a remote Nix store via SSH works: + + ```console + # nix store ping --store ssh://mac1 + ``` + +* Test whether a URL is a valid binary cache: + + ```console + # nix store ping --store https://cache.nixos.org + ``` + +* Test whether the Nix daemon is up and running: + + ```console + # nix store ping --store daemon + ``` + +# Description + +This command tests whether a particular Nix store (specified by the +argument `--store` *url*) can be accessed. What this means is +dependent on the type of the store. For instance, for an SSH store it +means that Nix can connect to the specified machine. + +)"" From a407d14339c2c480f0103a501bcd8a3373d935cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 18:34:52 +0100 Subject: [PATCH 230/477] Add 'nix eval' manpage --- src/nix/eval.cc | 27 +++--------------- src/nix/eval.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 src/nix/eval.md diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ea82e5300..321df7495 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -40,30 +40,11 @@ struct CmdEval : MixJSON, InstallableCommand return "evaluate a Nix expression"; } - Examples examples() override + std::string doc() override { - return { - { - "To evaluate a Nix expression given on the command line:", - "nix eval --expr '1 + 2'" - }, - { - "To evaluate a Nix expression from a file or URI:", - "nix eval -f ./my-nixpkgs hello.name" - }, - { - "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs#lib.version" - }, - { - "To print the store path of the Hello package:", - "nix eval --raw nixpkgs#hello" - }, - { - "To get a list of checks in the 'nix' flake:", - "nix eval nix#checks.x86_64-linux --apply builtins.attrNames" - }, - }; + return + #include "eval.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/eval.md b/src/nix/eval.md new file mode 100644 index 000000000..61334cde1 --- /dev/null +++ b/src/nix/eval.md @@ -0,0 +1,74 @@ +R""( + +# Examples + +* Evaluate a Nix expression given on the command line: + + ```console + # nix eval --expr '1 + 2' + ``` + +* Evaluate a Nix expression to JSON: + + ```console + # nix eval --json --expr '{ x = 1; }' + {"x":1} + ``` + +* Evaluate a Nix expression from a file: + + ```console + # nix eval -f ./my-nixpkgs hello.name + ``` + +* Get the current version of the `nixpkgs` flake: + + ```console + # nix eval --raw nixpkgs#lib.version + ``` + +* Print the store path of the Hello package: + + ```console + # nix eval --raw nixpkgs#hello + ``` + +* Get a list of checks in the `nix` flake: + + ```console + # nix eval nix#checks.x86_64-linux --apply builtins.attrNames + ``` + +* Generate a directory with the specified contents: + + ```console + # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }' + # cat ./out/foo + bar + # cat ./out/subdir/bla + 123 + +# Description + +This command evaluates the Nix expression *installable* and prints the +result on standard output. + +# Output format + +`nix eval` can produce output in several formats: + +* By default, the evaluation result is printed as a Nix expression. + +* With `--json`, the evaluation result is printed in JSON format. Note + that this fails if the result contains values that are not + representable as JSON, such as functions. + +* With `--raw`, the evaluation result must be a string, which is + printed verbatim, without any quoting. + +* With `--write-to` *path*, the evaluation result must be a string or + a nested attribute set whose leaf values are strings. These strings + are written to files named *path*/*attrpath*. *path* must not + already exist. + +)"" From 2cc02bbe7675acb4754b928b5c57fa316600e877 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 19:21:48 +0100 Subject: [PATCH 231/477] Add 'nix nar' manpages --- src/nix/cat.cc | 7 +++++++ src/nix/dump-path.cc | 11 ++++------- src/nix/ls.cc | 13 ++++++------- src/nix/nar-cat.md | 19 +++++++++++++++++++ src/nix/nar-dump-path.md | 17 +++++++++++++++++ src/nix/nar-ls.md | 24 ++++++++++++++++++++++++ src/nix/nar.cc | 9 ++++++++- src/nix/nar.md | 13 +++++++++++++ 8 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 src/nix/nar-cat.md create mode 100644 src/nix/nar-dump-path.md create mode 100644 src/nix/nar-ls.md create mode 100644 src/nix/nar.md diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 2ecffc9a5..fe2f0a241 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -62,6 +62,13 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } + std::string doc() override + { + return + #include "nar-cat.md" + ; + } + void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 256db64a9..63393ef9c 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -49,14 +49,11 @@ struct CmdDumpPath2 : Command return "serialise a path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To serialise directory 'foo' as a NAR:", - "nix nar dump-path ./foo" - }, - }; + return + #include "nar-dump-path.md" + ; } void run() override diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 1f5ed6913..df67240f9 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -75,6 +75,8 @@ struct MixLs : virtual Args, MixJSON if (json) { JSONPlaceholder jsonRoot(std::cout); + if (showDirectory) + throw UsageError("'--directory' is useless with '--json'"); listNar(jsonRoot, accessor, path, recursive); } else listText(accessor); @@ -127,14 +129,11 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } - Examples examples() override + std::string doc() override { - return { - Example{ - "To list a specific file in a NAR:", - "nix nar ls -l hello.nar /bin/hello" - }, - }; + return + #include "nar-ls.md" + ; } std::string description() override diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md new file mode 100644 index 000000000..55c481a28 --- /dev/null +++ b/src/nix/nar-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* List a file in a NAR and pipe it through `gunzip`: + + ```console + # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip + .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. + .TH HELLO "1" "November 2014" "hello 2.10" "User Commands" + … + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* inside the NAR file *nar*. + +)"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md new file mode 100644 index 000000000..26191ad25 --- /dev/null +++ b/src/nix/nar-dump-path.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* To serialise directory `foo` as a NAR: + + ```console + # nix nar dump-path ./foo > foo.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of +*path*, which must contain only regular files, directories and +symbolic links. The NAR is written to standard output. + +)"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md new file mode 100644 index 000000000..d373f9715 --- /dev/null +++ b/src/nix/nar-ls.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* To list a specific file in a NAR: + + ```console + # nix nar ls -l ./hello.nar /bin/hello + -r-xr-xr-x 38184 hello + ``` + +* To recursively list the contents of a directory inside a NAR, in JSON + format: + + ```console + # nix nar ls --json -R ./hello.nar /bin + {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} + ``` + +# Description + +This command shows information about a *path* inside NAR file *nar*. + +)"" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index e239ce96a..0775d3c25 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -9,7 +9,14 @@ struct CmdNar : NixMultiCommand std::string description() override { - return "query the contents of NAR files"; + return "create or inspect NAR files"; + } + + std::string doc() override + { + return + #include "nar.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/nar.md b/src/nix/nar.md new file mode 100644 index 000000000..a83b5c764 --- /dev/null +++ b/src/nix/nar.md @@ -0,0 +1,13 @@ +R""( + +# Description + +`nix nar` provides several subcommands for creating and inspecting +*Nix Archives* (NARs). + +# File format + +For the definition of the NAR file format, see Figure 5.2 in +https://edolstra.github.io/pubs/phd-thesis.pdf. + +)"" From c14ed3f8b2cbddb335227d2ff5188896e76b713f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 20:06:19 +0100 Subject: [PATCH 232/477] Add 'nix store' NAR-related manpages --- src/nix/cat.cc | 7 +++++++ src/nix/dump-path.cc | 11 ++++------- src/nix/ls.cc | 17 +++++++---------- src/nix/store-cat.md | 19 +++++++++++++++++++ src/nix/store-dump-path.md | 23 +++++++++++++++++++++++ src/nix/store-ls.md | 27 +++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 src/nix/store-cat.md create mode 100644 src/nix/store-dump-path.md create mode 100644 src/nix/store-ls.md diff --git a/src/nix/cat.cc b/src/nix/cat.cc index fe2f0a241..e28ee3c50 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -37,6 +37,13 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } + std::string doc() override + { + return + #include "store-cat.md" + ; + } + void run(ref store) override { cat(store->getFSAccessor()); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 63393ef9c..c4edc894b 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -11,14 +11,11 @@ struct CmdDumpPath : StorePathCommand return "serialise a store path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" - }, - }; + return + #include "store-dump-path.md" + ; } void run(ref store, const StorePath & storePath) override diff --git a/src/nix/ls.cc b/src/nix/ls.cc index df67240f9..d48287f27 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -94,21 +94,18 @@ struct CmdLsStore : StoreCommand, MixLs }); } - Examples examples() override - { - return { - Example{ - "To list the contents of a store path in a binary cache:", - "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" - }, - }; - } - std::string description() override { return "show information about a path in the Nix store"; } + std::string doc() override + { + return + #include "store-ls.md" + ; + } + void run(ref store) override { list(store->getFSAccessor()); diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md new file mode 100644 index 000000000..da2073473 --- /dev/null +++ b/src/nix/store-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Show the contents of a file in a binary cache: + + ```console + # nix store cat --store https://cache.nixos.org/ \ + /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1 + 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............| + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* in a Nix store. *path* can be a top-level store path or +any file inside a store path. + +)"" diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md new file mode 100644 index 000000000..4ef563526 --- /dev/null +++ b/src/nix/store-dump-path.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* To get a NAR containing the GNU Hello package: + + ```console + # nix store dump-path nixpkgs#hello > hello.nar + ``` + +* To get a NAR from the binary cache https://cache.nixos.org/: + + ```console + # nix store dump-path --store https://cache.nixos.org/ \ + /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of the +store path *installable*. The NAR is written to standard output. + +)"" diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md new file mode 100644 index 000000000..836efce42 --- /dev/null +++ b/src/nix/store-ls.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* To list the contents of a store path in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + dr-xr-xr-x 0 ./bin + -r-xr-xr-x 38184 ./bin/hello + dr-xr-xr-x 0 ./share + … + ``` + +* To show information about a specific file in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + -r-xr-xr-x 38184 hello + ``` + +# Description + +This command shows information about *path* in a Nix store. *path* can +be a top-level store path or any file inside a store path. + +)"" From 19540744ad164c13803048d5e8883fe68f9d2d10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 20:33:53 +0100 Subject: [PATCH 233/477] Add 'nix why-depends' manpage --- src/nix/why-depends.cc | 19 +++------- src/nix/why-depends.md | 80 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 src/nix/why-depends.md diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 57b9a2208..297b638cc 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -50,22 +50,11 @@ struct CmdWhyDepends : SourceExprCommand return "show why a package has another package in its closure"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs#hello nixpkgs#glibc" - }, - Example{ - "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11" - }, - Example{ - "To show why Glibc depends on itself:", - "nix why-depends nixpkgs#glibc nixpkgs#glibc" - }, - }; + return + #include "why-depends.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md new file mode 100644 index 000000000..dc13619e1 --- /dev/null +++ b/src/nix/why-depends.md @@ -0,0 +1,80 @@ +R""( + +# Examples + +* Show one path through the dependency graph leading from Hello to + Glibc: + + ```console + # nix why-depends nixpkgs#hello nixpkgs#glibc + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.… + → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32 + ``` + +* Show all files and paths in the dependency graph leading from + Thunderbird to libX11: + + ```console + # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11 + /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1 + ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf… + │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store… + │ → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5 + │ ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0… + │ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + … + ``` + +* Show why Glibc depends on itself: + + ```console + # nix why-depends nixpkgs#glibc nixpkgs#glibc + /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + └───lib/ld-2.31.so: …che Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache. --… + → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + ``` + +* Show why Geeqie has a build-time dependency on `systemd`: + + ```console + # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd + /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv + └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/… + → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv + └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix… + → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv + └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/… + → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv + ``` + +# Description + +Nix automatically determines potential runtime dependencies between +store paths by scanning for the *hash parts* of store paths. For +instance, if there exists a store path +`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file +inside another store path contains the string `9df65igw…`, then the +latter store path *refers* to the former, and thus might need it at +runtime. Nix always maintains the existence of the transitive closure +of a store path under the references relationship; it is therefore not +possible to install a store path without having all of its references +present. + +Sometimes Nix packages end up with unexpected runtime dependencies; +for instance, a reference to a compiler might accidentally end up in a +binary, causing the former to be in the latter's closure. This kind of +*closure size bloat* is undesirable. + +`nix why-depends` allows you to diagnose the cause of such issues. It +shows why the store path *package* depends on the store path +*dependency*, by showing a shortest sequence in the references graph +from the former to the latter. Also, for each node along this path, it +shows a file fragment containing a reference to the next store path in +the sequence. + +To show why derivation *package* has a build-time rather than runtime +dependency on derivation *dependency*, use `--derivation`. + +)"" From 6b32551aba5dfd6a912277297eb28cedc92da26d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 21:11:48 +0100 Subject: [PATCH 234/477] Add 'nix upgrade-nix' manpage --- src/nix/upgrade-nix.cc | 15 ++++----------- src/nix/upgrade-nix.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 src/nix/upgrade-nix.md diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 66ecc5b34..79be31e73 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand return "upgrade Nix to the latest stable version"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade Nix to the latest stable version:", - "nix upgrade-nix" - }, - Example{ - "To upgrade Nix in a specific profile:", - "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" - }, - }; + return + #include "upgrade-nix.md" + ; } Category category() override { return catNixInstallation; } diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md new file mode 100644 index 000000000..4d27daad9 --- /dev/null +++ b/src/nix/upgrade-nix.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Upgrade Nix to the latest stable version: + + ```console + # nix upgrade-nix + ``` + +* Upgrade Nix in a specific profile: + + ```console + # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + ``` + +# Description + +This command upgrades Nix to the latest version. By default, it +locates the directory containing the `nix` binary in the `$PATH` +environment variable. If that directory is a Nix profile, it will +upgrade the `nix` package in that profile to the latest stable binary +release. + +You cannot use this command to upgrade Nix in the system profile of a +NixOS system (that is, if `nix` is found in `/run/current-system`). + +)"" From 8dd7d7e9db8165c316b1ef168f57ed3632507fe2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 23:45:06 +0100 Subject: [PATCH 235/477] Add 'nix store verify' manpage --- src/nix/verify.cc | 15 ++++----------- src/nix/verify.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 src/nix/verify.md diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bcf85d7dd..16d42349f 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -35,18 +35,11 @@ struct CmdVerify : StorePathsCommand return "verify the integrity of store paths"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To verify the entire Nix store:", - "nix store verify --all" - }, - Example{ - "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix store verify -r -n2 --no-contents $(type -p firefox)" - }, - }; + return + #include "verify.md" + ; } void run(ref store, StorePaths storePaths) override diff --git a/src/nix/verify.md b/src/nix/verify.md new file mode 100644 index 000000000..1c43792e7 --- /dev/null +++ b/src/nix/verify.md @@ -0,0 +1,49 @@ +R""( + +# Examples + +* Verify the entire Nix store: + + ```console + # nix store verify --all + ``` + +* Check whether each path in the closure of Firefox has at least 2 + signatures: + + ```console + # nix store verify -r -n2 --no-contents $(type -p firefox) + ``` + +* Verify a store path in the binary cache `https://cache.nixos.org/`: + + ```console + # nix store verify --store https://cache.nixos.org/ \ + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +# Description + +This command verifies the integrity of the store paths *installables*, +or, if `--all` is given, the entire Nix store. For each path, it +checks that + +* its contents match the NAR hash recorded in the Nix database; and + +* it is *trusted*, that is, it is signed by at least one trusted + signing key, is content-addressed, or is built locally ("ultimately + trusted"). + +# Exit status + +The exit status of this command is the sum of the following values: + +* **1** if any path is corrupted (i.e. its contents don't match the + recorded NAR hash). + +* **2** if any path is untrusted. + +* **4** if any path couldn't be verified for any other reason (such as + an I/O error). + +)"" From cb25a89f1cb9b3a26d84b3429b309be2cfa513a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 23:54:11 +0100 Subject: [PATCH 236/477] Add 'nix store optimise' manpage --- src/nix/optimise-store.cc | 11 ++++------- src/nix/optimise-store.md | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 src/nix/optimise-store.md diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index bc7f175ac..985006e5a 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -13,14 +13,11 @@ struct CmdOptimiseStore : StoreCommand return "replace identical files in the store by hard links"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To optimise the Nix store:", - "nix store optimise" - }, - }; + return + #include "optimise-store.md" + ; } void run(ref store) override diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md new file mode 100644 index 000000000..f6fb66f97 --- /dev/null +++ b/src/nix/optimise-store.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Optimise the Nix store: + + ```console + nix store optimise + ``` + +# Description + +This command deduplicates the Nix store: it scans the store for +regular files with identical contents, and replaces them with hard +links to a single instance. + +Note that you can also set `auto-optimise-store` to `true` in +`nix.conf` to perform this optimisation incrementally whenever a new +path is added to the Nix store. To make this efficient, Nix maintains +a content-addressed index of all the files in the Nix store in the +directory `/nix/store/.links/`. + +)"" From 2e599dbb88855311c33c70460b82ec16487c9071 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 13:45:42 +0100 Subject: [PATCH 237/477] Add 'nix path-info' manpage --- src/nix/path-info.cc | 35 +++-------------- src/nix/path-info.md | 94 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 src/nix/path-info.md diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 63cf885f9..30b6a50f8 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -29,38 +29,15 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } - Category category() override { return catSecondary; } - - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", - "nix path-info -rS /run/current-system | sort -nk2" - }, - Example{ - "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs#rust" - }, - Example{ - "To check the existence of a path in a binary cache:", - "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" - }, - Example{ - "To print the 10 most recently added paths (using --json and the jq(1) command):", - "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" - }, - Example{ - "To show the size of the entire Nix store:", - "nix path-info --json --all | jq 'map(.narSize) | add'" - }, - Example{ - "To show every path whose closure is bigger than 1 GB, sorted by closure size:", - "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" - }, - }; + return + #include "path-info.md" + ; } + Category category() override { return catSecondary; } + void printSize(uint64_t value) { if (!humanReadable) { diff --git a/src/nix/path-info.md b/src/nix/path-info.md new file mode 100644 index 000000000..b4ba5862d --- /dev/null +++ b/src/nix/path-info.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Print the store path produced by `nixpkgs#hello`: + + ```console + # nix path-info nixpkgs#hello + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +* Show the closure sizes of every path in the current NixOS system + closure, sorted by size: + + ```console + # nix path-info -rS /run/current-system | sort -nk2 + /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 + /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 + … + /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc 5783255504 + /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65 5887562256 + ``` + +* Show a package's closure size and all its dependencies with human + readable sizes: + + ```console + # nix path-info -rsSh nixpkgs#rustc + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + … + ``` + +* Check the existence of a path in a binary cache: + + ```console + # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid + + ``` + +* Print the 10 most recently added paths (using --json and the jq(1) + command): + + ```console + # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + ``` + +* Show the size of the entire Nix store: + + ```console + # nix path-info --json --all | jq 'map(.narSize) | add' + 49812020936 + ``` + +* Show every path whose closure is bigger than 1 GB, sorted by closure + size: + + ```console + # nix path-info --json --all -S \ + | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + [ + …, + [ + "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + 5887562256 + ] + ] + ``` + +* Print the path of the store derivation produced by `nixpkgs#hello`: + + ```console + # nix path-info --derivation nixpkgs#hello + /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv + ``` + +# Description + +This command shows information about the store paths produced by +*installables*, or about all paths in the store if you pass `--all`. + +By default, this command only prints the store paths. You can get +additional information by passing flags such as `--closure-size`, +--size`, `--sigs` or `--json`. + +> **Warning** +> +> Note that `nix path-info` does not build or substitute the +> *installables* you specify. Thus, if the corresponding store paths +> don't already exist, this command will fail. You can use `nix build` +> to ensure that they exist. + +)"" From cdf20e04b7acc0efd7fa9640283103502ac80c53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 14:14:30 +0100 Subject: [PATCH 238/477] Doh --- src/nix/path-info.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/path-info.md b/src/nix/path-info.md index b4ba5862d..76a83e39d 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -73,7 +73,7 @@ R""( ```console # nix path-info --derivation nixpkgs#hello /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv - ``` + ``` # Description From e6bea9c9b10ded0e65981edf84cedd00ec86883a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 17:36:59 +0100 Subject: [PATCH 239/477] Add 'nix store make-content-addressable' manpage --- src/nix/make-content-addressable.cc | 17 +++------ src/nix/make-content-addressable.md | 59 +++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 src/nix/make-content-addressable.md diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5165c4804..f5bdc7e65 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -15,21 +15,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON std::string description() override { - return "rewrite a path or closure to content-addressable form"; + return "rewrite a path or closure to content-addressed form"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix store make-content-addressable nixpkgs#hello" - }, - Example{ - "To compute a content-addressable representation of the current NixOS system closure:", - "nix store make-content-addressable -r /run/current-system" - }, - }; + return + #include "make-content-addressable.md" + ; } void run(ref store, StorePaths storePaths) override diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md new file mode 100644 index 000000000..3dd847edc --- /dev/null +++ b/src/nix/make-content-addressable.md @@ -0,0 +1,59 @@ +R""( + +# Examples + +* Create a content-addressed representation of the closure of GNU Hello: + + ```console + # nix store make-content-addressable -r nixpkgs#hello + … + rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' + ``` + + Since the resulting paths are content-addressed, they are always + trusted and don't need signatures to copied to another store: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 + ``` + + By contrast, the original closure is input-addressed, so it does + need signatures to be trusted: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + ``` + +* Create a content-addressed representation of the current NixOS + system closure: + + ```console + # nix store make-content-addressable -r /run/current-system + ``` + +# Description + +This command converts the closure of the store paths specified by +*installables* to content-addressed form. Nix store paths are usually +*input-addressed*, meaning that the hash part of the store path is +computed from the contents of the derivation (i.e., the build-time +dependency graph). Input-addressed paths need to be signed by a +trusted key if you want to import them into a store, because we need +to trust that the contents of the path were actually built by the +derivation. + +By contrast, in a *content-addressed* path, the hash part is computed +from the contents of the path. This allows the contents of the path to +be verified without any additional information such as +signatures. This means that a command like + +```console +# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ + --substituters https://my-cache.example.org +``` + +will succeed even if the binary cache `https://my-cache.example.org` +doesn't present any signatures. + +)"" From daf365b0b731bb3ac86128a965394dcff8d6f5b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 18:40:16 +0100 Subject: [PATCH 240/477] Add 'nix help' manpage --- src/nix/help.md | 17 +++++++++++++++++ src/nix/main.cc | 17 +++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 src/nix/help.md diff --git a/src/nix/help.md b/src/nix/help.md new file mode 100644 index 000000000..734f35028 --- /dev/null +++ b/src/nix/help.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* Show help about `nix` in general: + + ```console + # nix help + ``` + +* Show help about a particular subcommand: + + ```console + # nix help flake info + ``` + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index e7a15dec9..afe7cb8d7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -205,21 +205,14 @@ struct CmdHelp : Command std::string description() override { - return "show help about 'nix' or a particular subcommand"; + return "show help about `nix` or a particular subcommand"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show help about 'nix' in general:", - "nix help" - }, - Example{ - "To show help about a particular subcommand:", - "nix help run" - }, - }; + return + #include "help.md" + ; } void run() override From 3b123a6ee63aef29d61a3dac7f2f01c2e92cd6d0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:13:23 +0100 Subject: [PATCH 241/477] nix show-derivation: Say "system" instead of "platform" There is really no good reason to use "platform" except that that's what we use internally (also for no good reason). --- src/nix/show-derivation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 8e1a58ac2..211e6a27a 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -103,7 +103,7 @@ struct CmdShowDerivation : InstallablesCommand } } - drvObj.attr("platform", drv.platform); + drvObj.attr("system", drv.platform); drvObj.attr("builder", drv.builder); { From 4f3e7f4eec9ef5fb86aea9f745a3574cc5cfae28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:14:18 +0100 Subject: [PATCH 242/477] Add 'nix show-derivation' manpage --- src/nix/show-derivation.cc | 15 ++---- src/nix/show-derivation.md | 103 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 src/nix/show-derivation.md diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 211e6a27a..13f2c8e69 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -29,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand return "show the contents of a store derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs#hello" - }, - Example{ - "To show the full derivation graph (if available) that produced your NixOS system:", - "nix show-derivation -r /run/current-system" - }, - }; + return + #include "show-derivation.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md new file mode 100644 index 000000000..aa863899c --- /dev/null +++ b/src/nix/show-derivation.md @@ -0,0 +1,103 @@ +R""( + +# Examples + +* Show the store derivation that results from evaluating the Hello + package: + + ```console + # nix show-derivation nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix show-derivation -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix show-derivation -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +store derivations to which *installables* evaluate. Store derivations +are used internally by Nix. They are store paths with extension `.drv` +that represent the build-time dependency graph to which a Nix +expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" From f4e9d4fcb3e393af2736f28fc41e4e3b79a8e60d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:58:04 +0100 Subject: [PATCH 243/477] Add 'nix store diff-closures' manpage --- src/nix/diff-closures.cc | 11 ++++----- src/nix/diff-closures.md | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/nix/diff-closures.md diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index f72b5eff7..0c7d531c1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -121,14 +121,11 @@ struct CmdDiffClosures : SourceExprCommand return "show what packages and versions were added and removed between two closures"; } - Examples examples() override + std::string doc() override { - return { - { - "To show what got added and removed between two versions of the NixOS system profile:", - "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", - }, - }; + return + #include "diff-closures.md" + ; } void run(ref store) override diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md new file mode 100644 index 000000000..0294c0d8d --- /dev/null +++ b/src/nix/diff-closures.md @@ -0,0 +1,51 @@ +R""( + +# Examples + +* Show what got added and removed between two versions of the NixOS + system profile: + + ```console + # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link + acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18 + baloo-widgets: 20.08.1 → 20.08.2 + bluez-qt: +12.6 KiB + dolphin: 20.08.1 → 20.08.2, +13.9 KiB + kdeconnect: 20.08.2 → ∅, -6597.8 KiB + kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + … + ``` + +# Description + +This command shows the differences between the two closures *before* +and *after* with respect to the addition, removal, or version change +of packages, as well as changes in store path sizes. + +For each package name in the two closures (where a package name is +defined as the name component of a store path excluding the version), +if there is a change in the set of versions of the package, or a +change in the size of the store paths of more than 8 KiB, it prints a +line like this: + +```console +dolphin: 20.08.1 → 20.08.2, +13.9 KiB +``` + +No size change is shown if it's below the threshold. If the package +does not exist in either the *before* or *after* closures, it is +represented using `∅` (empty set) on the appropriate side of the +arrow. If a package has an empty version string, the version is +rendered as `ε` (epsilon). + +There may be multiple versions of a package in each closure. In that +case, only the changed versions are shown. Thus, + +```console +libfoo: 1.2, 1.3 → 1.4 +``` + +leaves open the possibility that there are other versions (e.g. `1.1`) +that exist in both closures. + +)"" From 0c09f63de84e15c15e8621a53d6d25f023b4ad06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Dec 2020 11:45:59 +0100 Subject: [PATCH 244/477] Add 'nix bundle' manpage Fixes #4375. --- src/nix/bundle.cc | 11 ++++------- src/nix/bundle.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/nix/bundle.md diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index eddd82f40..5f558b01e 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -40,14 +40,11 @@ struct CmdBundle : InstallableCommand return "bundle an application so that it works outside of the Nix store"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To bundle Hello:", - "nix bundle hello" - }, - }; + return + #include "bundle.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/bundle.md b/src/nix/bundle.md new file mode 100644 index 000000000..c183a170d --- /dev/null +++ b/src/nix/bundle.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Bundle Hello: + + ```console + # nix bundle nixpkgs#hello + # ./hello + Hello, world! + ``` + +* Bundle a specific version of Nix: + + ```console + # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec + # ./nix --version + nix (Nix) 2.4pre20201215_e3ddffb + ``` + +# Description + +`nix bundle` packs the closure of the [Nix app](./nix3-run.md) +*installable* into a single self-extracting executable. See the +[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle) +for more details. + +> **Note** +> +> This command only works on Linux. + +)"" From 16e34085e8f45758b97c41cfcd720552c68a3c98 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Dec 2020 14:25:36 +0100 Subject: [PATCH 245/477] Add 'nix profile' manpage --- src/nix/profile-diff-closures.md | 28 ++++++++ src/nix/profile-info.md | 31 +++++++++ src/nix/profile-install.md | 27 ++++++++ src/nix/profile-remove.md | 32 +++++++++ src/nix/profile-upgrade.md | 41 ++++++++++++ src/nix/profile.cc | 90 +++++++++----------------- src/nix/profile.md | 107 +++++++++++++++++++++++++++++++ 7 files changed, 295 insertions(+), 61 deletions(-) create mode 100644 src/nix/profile-diff-closures.md create mode 100644 src/nix/profile-info.md create mode 100644 src/nix/profile-install.md create mode 100644 src/nix/profile-remove.md create mode 100644 src/nix/profile-upgrade.md create mode 100644 src/nix/profile.md diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md new file mode 100644 index 000000000..295d1252b --- /dev/null +++ b/src/nix/profile-diff-closures.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Show what changed between each version of the NixOS system + profile: + + ```console + # nix profile diff-closures --profile /nix/var/nix/profiles/system + Version 13 -> 14: + acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14 + aws-sdk-cpp: -6723.1 KiB + … + + Version 14 -> 15: + acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16 + attica: -996.2 KiB + breeze-icons: -78713.5 KiB + brotli: 1.0.7 → 1.0.9, +44.2 KiB + ``` + +# Description + +This command shows the difference between the closures of subsequent +versions of a profile. See [`nix store +diff-closures`](nix3-store-diff-closures.md) for details. + +)"" diff --git a/src/nix/profile-info.md b/src/nix/profile-info.md new file mode 100644 index 000000000..a0c04fc8c --- /dev/null +++ b/src/nix/profile-info.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Show what packages are installed in the default profile: + + ```console + # nix profile info + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 + 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 + 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + ``` + +# Description + +This command shows what packages are currently installed in a +profile. The output consists of one line per package, with the +following fields: + +* An integer that can be used to unambiguously identify the package in + invocations of `nix profile remove` and `nix profile upgrade`. + +* The original ("mutable") flake reference and output attribute path + used at installation time. + +* The immutable flake reference to which the mutable flake reference + was resolved. + +* The store path(s) of the package. + +)"" diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md new file mode 100644 index 000000000..e3009491e --- /dev/null +++ b/src/nix/profile-install.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* Install a package from Nixpkgs: + + ```console + # nix profile install nixpkgs#hello + ``` + +* Install a package from a specific branch of Nixpkgs: + + ```console + # nix profile install nixpkgs/release-20.09#hello + ``` + +* Install a package from a specific revision of Nixpkgs: + + ```console + # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello + ``` + +# Description + +This command adds *installables* to a Nix profile. + +)"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md new file mode 100644 index 000000000..dcf825da9 --- /dev/null +++ b/src/nix/profile-remove.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Remove a package by position: + + ```console + # nix profile remove 3 + ``` + +* Remove a package by attribute path: + + ```console + # nix profile remove packages.x86_64-linux.hello + ``` + +* Remove all packages: + ```console + # nix profile remove '.*' + ``` + +* Remove a package by store path: + + ```console + # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10 + ``` + +# Description + +This command removes a package from a profile. + +)"" diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md new file mode 100644 index 000000000..2bd5d256d --- /dev/null +++ b/src/nix/profile-upgrade.md @@ -0,0 +1,41 @@ +R""( + +# Examples + +* Upgrade all packages that were installed using a mutable flake + reference: + + ```console + # nix profile upgrade '.*' + ``` + +* Upgrade a specific package: + + ```console + # nix profile upgrade packages.x86_64-linux.hello + ``` + +* Upgrade a specific profile element by number: + + ```console + # nix profile info + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … + + # nix profile upgrade 0 + ``` + +# Description + +This command upgrades a previously installed package in a Nix profile, +by fetching and evaluating the latest version of the flake from which +the package was installed. + +> **Warning** +> +> This only works if you used a *mutable* flake reference at +> installation time, e.g. `nixpkgs#hello`. It does not work if you +> used an *immutable* flake reference +> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), +> since in that case the "latest version" is always the same. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cf5ccd62..d8d2b3a70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -151,22 +151,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile return "install a package into a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To install a package from Nixpkgs:", - "nix profile install nixpkgs#hello" - }, - Example{ - "To install a package from a specific branch of Nixpkgs:", - "nix profile install nixpkgs/release-19.09#hello" - }, - Example{ - "To install a package from a specific revision of Nixpkgs:", - "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" - }, - }; + return + #include "profile-install.md" + ; } void run(ref store) override @@ -257,26 +246,11 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem return "remove packages from a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To remove a package by attribute path:", - "nix profile remove packages.x86_64-linux.hello" - }, - Example{ - "To remove all packages:", - "nix profile remove '.*'" - }, - Example{ - "To remove a package by store path:", - "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" - }, - Example{ - "To remove a package by position:", - "nix profile remove 3" - }, - }; + return + #include "profile-remove.md" + ; } void run(ref store) override @@ -310,18 +284,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf return "upgrade packages using their most recent flake"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade all packages that were installed using a mutable flake reference:", - "nix profile upgrade '.*'" - }, - Example{ - "To upgrade a specific package:", - "nix profile upgrade packages.x86_64-linux.hello" - }, - }; + return + #include "profile-upgrade.md" + ; } void run(ref store) override @@ -377,14 +344,11 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro return "list installed packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what packages are installed in the default profile:", - "nix profile info" - }, - }; + return + #include "profile-info.md" + ; } void run(ref store) override @@ -405,17 +369,14 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile { std::string description() override { - return "show the closure difference between each generation of a profile"; + return "show the closure difference between each version of a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what changed between each generation of the NixOS system profile:", - "nix profile diff-closures --profile /nix/var/nix/profiles/system" - }, - }; + return + #include "profile-diff-closures.md" + ; } void run(ref store) override @@ -429,7 +390,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile if (prevGen) { if (!first) std::cout << "\n"; first = false; - std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number); + std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -458,6 +419,13 @@ struct CmdProfile : NixMultiCommand return "manage Nix profiles"; } + std::string doc() override + { + return + #include "profile.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/profile.md b/src/nix/profile.md new file mode 100644 index 000000000..d3ddcd3d1 --- /dev/null +++ b/src/nix/profile.md @@ -0,0 +1,107 @@ +R""( + +# Description + +`nix profile` allows you to create and manage *Nix profiles*. A Nix +profile is a set of packages that can be installed and upgraded +independently from each other. Nix profiles are versioned, allowing +them to be rolled back easily. + +# Default profile + +The default profile used by `nix profile` is `$HOME/.nix-profile`, +which, if it does not exist, is created as a symlink to +`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. + +You can specify another profile location using `--profile` *path*. + +# Filesystem layout + +Profiles are versioned as follows. When using profile *path*, *path* +is a symlink to *path*`-`*N*, where *N* is the current *version* of +the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix +store. For example: + +```console +$ ls -l /nix/var/nix/profiles/per-user/alice/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ +/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +The file `manifest.json` records the provenance of the packages that +are installed in this version of the profile. It looks like this: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUri": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUri`: The [flake reference](./nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The immutable flake reference to which `originalUri` + resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `defaultPackage.x86_64-linux`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. + +)"" From 629af83b2d86d77305dc994b83f176a377106c3e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 14 Jul 2020 20:59:24 +0200 Subject: [PATCH 246/477] Provide a more meaningful error-message for `builtins.fetchGit` if a revision can't be checked out A common pitfall when using e.g. `builtins.fetchGit` is the `fatal: not a tree object`-error when trying to fetch a revision of a git-repository that isn't on the `master` branch and no `ref` is specified. In order to make clear what's the problem, I added a simple check whether the revision in question exists and if it doesn't a more meaningful error-message is displayed: ``` nix-repl> builtins.fetchGit { url = "https://github.com/owner/myrepo"; rev = ""; } moderror: --- Error -------------------------------------------------------------------- nix Cannot find Git revision 'bf1cc5c648e6aed7360448a3745bb2fe4fbbf0e9' in ref 'master' of repository 'https://gitlab.com/Ma27/nvim.nix'! Please make sure that the rev exists on the ref you've specified or add allRefs = true; to fetchGit. ``` Closes #2431 --- src/libfetchers/git.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e7712c5fd..1f298c2d6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -392,6 +392,28 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + RunOptions checkCommitOpts( + "git", + { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() } + ); + checkCommitOpts.searchPath = true; + checkCommitOpts.mergeStderrToStdout = true; + + auto result = runProgram(checkCommitOpts); + if (WEXITSTATUS(result.first) == 128 + && result.second.find("bad file") != std::string::npos + ) { + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + input.getRev()->gitRev(), + *input.getRef(), + actualUrl + ); + } + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); From 2857b1baaf78bbadeec01adfae8a50fc0f2a254f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 17 Jul 2020 20:34:57 +0200 Subject: [PATCH 247/477] Add explicit `allRefs = true;` argument to `fetchGit` Sometimes it's necessary to fetch a git repository at a revision and it's unknown which ref contains the revision in question. An example would be a Cargo.lock which only provides the URL and the revision when using a git repository as build input. However it's considered a bad practice to perform a full checkout of a repository since this may take a lot of time and can eat up a lot of disk space. This patch makes a full checkout explicit by adding an `allRefs` argument to `builtins.fetchGit` which fetches all refs if explicitly set to true. Closes #2409 --- src/libfetchers/git.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f298c2d6..81c647f89 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); + maybeGetBoolAttr(attrs, "allRefs"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -169,10 +170,12 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; if (submodules) cacheType += "-submodules"; + if (allRefs) cacheType += "-all-refs"; auto getImmutableAttrs = [&]() { @@ -338,11 +341,15 @@ struct GitInputScheme : InputScheme } } } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + if (allRefs) { + doFetch = true; + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } } if (doFetch) { @@ -352,9 +359,11 @@ struct GitInputScheme : InputScheme // we're using --quiet for now. Should process its stderr. try { auto ref = input.getRef(); - auto fetchRef = ref->compare(0, 5, "refs/") == 0 - ? *ref - : "refs/heads/" + *ref; + auto fetchRef = allRefs + ? "refs/*" + : ref->compare(0, 5, "refs/") == 0 + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; From 724b7f4fb660212a97ba6482208c299158720c5b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 11:15:29 +0100 Subject: [PATCH 248/477] Don't log from inside the logger This deadlocks ProgressBar, e.g. # nix run --impure --no-substitute --store '/tmp/nix2?store=/foo' --expr 'derivation { builder = /nix/store/zi90rxslsm4mlr46l2xws1rm94g7pk8p-busybox-1.31.1-x86_64-unknown-linux-musl/bin/busybox; }' leads to Thread 1 (Thread 0x7ffff6126e80 (LWP 12250)): #0 0x00007ffff7215d62 in __lll_lock_wait () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0 #1 0x00007ffff720e721 in pthread_mutex_lock () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0 #2 0x00007ffff7ad17fa in __gthread_mutex_lock (__mutex=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/x86_64-unknown-linux-gnu/bits/gthr-default.h:749 #3 std::mutex::lock (this=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/std_mutex.h:100 #4 std::unique_lock::lock (this=0x7fffffff09a8, this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:141 #5 std::unique_lock::unique_lock (__m=..., this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:71 #6 nix::Sync::Lock::Lock (s=0x6c5448, this=0x7fffffff09a0) at src/libutil/sync.hh:45 #7 nix::Sync::lock (this=0x6c5448) at src/libutil/sync.hh:85 #8 nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:131 #9 0x00007ffff7608cfd in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88 #10 nix::getCodeLines (errPos=...) at src/libutil/error.cc:66 #11 0x00007ffff76073f2 in nix::showErrorInfo (out=..., einfo=..., showTrace=) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/optional:897 #12 0x00007ffff7ad19e7 in nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:134 #13 0x00007ffff7ab9d10 in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88 #14 nix::handleExceptions(std::__cxx11::basic_string, std::allocator > const&, std::function) (programName="/home/eelco/Dev/nix/outputs/out/bin/nix", fun=...) at src/libmain/shared.cc:328 #15 0x000000000046226b in main (argc=, argv=) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/ext/new_allocator.h:80 --- src/libutil/error.cc | 50 +++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 803a72953..e7dc3f1d3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -62,35 +62,28 @@ std::optional getCodeLines(const ErrPos & errPos) LinesOfCode loc; try { AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) { - logError(SysError("opening file '%1%'", errPos.file).info()); - return std::nullopt; - } - else + if (!fd) return {}; + + // count the newlines. + int count = 0; + string line; + int pl = errPos.line - 1; + do { - // count the newlines. - int count = 0; - string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - { - ; - } - else if (count == pl) { - loc.prevLineOfCode = line; - } else if (count == pl + 1) { - loc.errLineOfCode = line; - } else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } + line = readLine(fd.get()); + ++count; + if (count < pl) + ; + else if (count == pl) + loc.prevLineOfCode = line; + else if (count == pl + 1) + loc.errLineOfCode = line; + else if (count == pl + 2) { + loc.nextLineOfCode = line; + break; + } + } while (true); + return loc; } catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) @@ -99,7 +92,6 @@ std::optional getCodeLines(const ErrPos & errPos) return std::nullopt; } catch (std::exception & e) { - printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } } else { From e54971d019821dd213db028a80cd5fca85ee2ed6 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 8 Sep 2020 19:45:28 +0200 Subject: [PATCH 249/477] Document `allRefs` argument of `builtins.fetchTree` --- src/libexpr/primops/fetchTree.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6e7ddde8e..133299030 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -324,6 +324,11 @@ static RegisterPrimOp primop_fetchGit({ A Boolean parameter that specifies whether submodules should be checked out. Defaults to `false`. + - allRefs + Whether to fetch all refs of the repository. With this argument being + true, it's possible to load a `rev` from *any* `ref` (by default only + `rev`s from the specified `ref` are supported). + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: From 897ae235fc2cef0ce711470a7b620241d82a1b09 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 22 Dec 2020 12:18:10 +0100 Subject: [PATCH 250/477] tests/fetchGit: test behavior of `allRefs = true;` --- tests/fetchGit.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 76390fa59..1e8963d76 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -41,6 +41,19 @@ export _NIX_FORCE_HTTP=1 path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] +# Fetch a rev from another branch +git -C $repo checkout -b devtest +echo "different file" >> $TEST_ROOT/git/differentbranch +git -C $repo add differentbranch +git -C $repo commit -m 'Test2' +git -C $repo checkout master +devrev=$(git -C $repo rev-parse devtest) +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'Cannot find Git revision' ]] + +[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] + # In pure eval mode, fetchGit without a revision should fail. [[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") = world ]] (! nix eval --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") From 5373f4be3b1427ed73303448a1fa801726f4dfa0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 12:28:50 +0100 Subject: [PATCH 251/477] chrootHelper: Handle symlinks in the root directory This is necessary on Ubuntu where /bin and /lib* are symlinks. --- src/nix/run.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nix/run.cc b/src/nix/run.cc index 92a52c6cd..ec61fc79a 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -258,14 +258,16 @@ void chrootHelper(int argc, char * * argv) for (auto entry : readDirectory("/")) { auto src = "/" + entry.name; - auto st = lstat(src); - if (!S_ISDIR(st.st_mode)) continue; Path dst = tmpDir + "/" + entry.name; if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); - if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); + auto st = lstat(src); + if (S_ISDIR(st.st_mode)) { + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } else if (S_ISLNK(st.st_mode)) + createSymlink(readLink(src), dst); } char * cwd = getcwd(0, 0); From 75efa421340b8fb2be6cf5351d3ef36a93b294e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 14:43:20 +0100 Subject: [PATCH 252/477] Move into the nix binary This makes the statically linked nix binary just work, without needing any additional files. --- Makefile | 1 - corepkgs/local.mk | 4 ---- src/libexpr/eval.cc | 7 ++----- src/libexpr/eval.hh | 2 ++ {corepkgs => src/libexpr}/fetchurl.nix | 0 src/libexpr/local.mk | 2 +- src/libexpr/parser.y | 4 ++++ src/libexpr/primops.cc | 10 +++++++++- tests/fetchurl.sh | 14 +++++++------- tests/lang/eval-okay-search-path.nix | 5 ++--- 10 files changed, 27 insertions(+), 22 deletions(-) delete mode 100644 corepkgs/local.mk rename {corepkgs => src/libexpr}/fetchurl.nix (100%) diff --git a/Makefile b/Makefile index c50d2c40f..f80b8bb82 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ makefiles = \ src/nix/local.mk \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ - corepkgs/local.mk \ misc/bash/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ diff --git a/corepkgs/local.mk b/corepkgs/local.mk deleted file mode 100644 index 0bc91cfab..000000000 --- a/corepkgs/local.mk +++ /dev/null @@ -1,4 +0,0 @@ -corepkgs_FILES = \ - fetchurl.nix - -$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5a641d02c..ead5bf8c7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -402,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref store) for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - try { - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - } catch (Error &) { - } - if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); @@ -457,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_) */ Path abspath = canonPath(path_); + if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e1f61baa..e3eaed6d3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -432,4 +432,6 @@ struct EvalSettings : Config extern EvalSettings evalSettings; +static const std::string corepkgsPrefix{"/__corepkgs__/"}; + } diff --git a/corepkgs/fetchurl.nix b/src/libexpr/fetchurl.nix similarity index 100% rename from corepkgs/fetchurl.nix rename to src/libexpr/fetchurl.nix diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 519da33f7..26c53d301 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -40,6 +40,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index a4c84c526..85eb05d61 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -698,6 +698,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } + + if (hasPrefix(path, "nix/")) + return corepkgsPrefix + path.substr(4); + throw ThrownError({ .hint = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 18438f681..c73a94f4e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -164,7 +164,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS state.forceFunction(**fun, pos); mkApp(v, **fun, w); state.forceAttrs(v, pos); - } else { + } + + else if (path == corepkgsPrefix + "fetchurl.nix") { + state.eval(state.parseExprFromString( + #include "fetchurl.nix.gen.hh" + , "/"), v); + } + + else { if (!vScope) state.evalFile(realPath, v); else { diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 10ec0173a..cd84e9a4c 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,7 +5,7 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -14,7 +14,7 @@ clearStore hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -25,7 +25,7 @@ hash=$(nix hash file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) cmp $outPath fetchurl.sh @@ -38,10 +38,10 @@ hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store store add-file ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ +nix-build --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ --no-out-link --substituters $other_store # Test unpacking a NAR. @@ -55,7 +55,7 @@ nix-store --dump $TEST_ROOT/archive > $nar hash=$(nix-hash --flat --type sha256 $nar) -outPath=$(nix-build '' --argstr url file://$nar --argstr sha256 $hash \ +outPath=$(nix-build --expr 'import ' --argstr url file://$nar --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) echo $outPath | grep -q 'xyzzy' @@ -69,7 +69,7 @@ nix-store --delete $outPath narxz=$TEST_ROOT/archive.nar.xz rm -f $narxz xz --keep $nar -outPath=$(nix-build '' --argstr url file://$narxz --argstr sha256 $hash \ +outPath=$(nix-build --expr 'import ' --argstr url file://$narxz --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) test -x $outPath/fetchurl.sh diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index c5a123d04..6fe33decc 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -1,10 +1,9 @@ with import ./lib.nix; with builtins; -assert pathExists ; +assert isFunction (import ); -assert length __nixPath == 6; -assert length (filter (x: x.prefix == "nix") __nixPath) == 1; +assert length __nixPath == 5; assert length (filter (x: baseNameOf x.path == "dir4") __nixPath) == 1; import + import + import + import From e27044216bf597710893e0366dbff60efbdaf0a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 16:23:57 +0100 Subject: [PATCH 253/477] Fix tests --- src/libutil/tests/logging.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..5b32c84a4 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -49,7 +49,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); + ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); } TEST(logEI, appendingHintsToPreviousError) { @@ -208,7 +208,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } TEST(logError, logErrorWithOnlyHintAndName) { @@ -290,7 +290,7 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); } TEST(addTrace, hideTracesWithoutShowTrace) { From c9279b831e91a762851464826eaa8a8e30979578 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 13:19:53 +0100 Subject: [PATCH 254/477] Add 'nix flake' manpages --- src/nix/flake-archive.md | 29 ++ src/nix/flake-check.md | 68 +++++ src/nix/flake-clone.md | 18 ++ src/nix/flake-info.md | 99 ++++++ src/nix/flake-init.md | 54 ++++ src/nix/flake-list-inputs.md | 23 ++ src/nix/flake-new.md | 34 +++ src/nix/flake-show.md | 38 +++ src/nix/flake-update.md | 53 ++++ src/nix/flake.cc | 94 ++++-- src/nix/flake.md | 566 +++++++++++++++++++++++++++++++++++ 11 files changed, 1046 insertions(+), 30 deletions(-) create mode 100644 src/nix/flake-archive.md create mode 100644 src/nix/flake-check.md create mode 100644 src/nix/flake-clone.md create mode 100644 src/nix/flake-info.md create mode 100644 src/nix/flake-init.md create mode 100644 src/nix/flake-list-inputs.md create mode 100644 src/nix/flake-new.md create mode 100644 src/nix/flake-show.md create mode 100644 src/nix/flake-update.md create mode 100644 src/nix/flake.md diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md new file mode 100644 index 000000000..85bbeeb16 --- /dev/null +++ b/src/nix/flake-archive.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Copy the `dwarffs` flake and its dependencies to a binary cache: + + ```console + # nix flake archive --to file:///tmp/my-cache dwarffs + ``` + +* Fetch the `dwarffs` flake and its dependencies to the local Nix + store: + + ```console + # nix flake archive dwarffs + ``` + +* Print the store paths of the flake sources of NixOps without + fetching them: + + ```console + # nix flake archive --json --dry-run nixops + ``` + +# Description + +FIXME + +)"" diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md new file mode 100644 index 000000000..dc079ba0c --- /dev/null +++ b/src/nix/flake-check.md @@ -0,0 +1,68 @@ +R""( + +# Examples + +* Evaluate the flake in the current directory, and build its checks: + + ```console + # nix flake check + ``` + +* Verify that the `patchelf` flake evaluates, but don't build its + checks: + + ```console + # nix flake check --no-build github:NixOS/patchelf + ``` + +# Description + +This command verifies that the flake specified by flake reference +*flake-url* can be evaluated successfully (as detailed below), and +that the derivations specified by the flake's `checks` output can be +built successfully. + +# Evaluation checks + +This following flake output attributes must be derivations: + +* `checks.`*system*`.`*name* +* `defaultPackage.`*system*` +* `devShell.`*system*` +* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `packages.`*system*`.`*name* + +The following flake output attributes must be [app +definitions](./nix3-run.md): + +* `apps.`*system*`.`*name* +* `defaultApp.`*system*` + +The following flake output attributes must be [template +definitions](./nix3-flake-init.md): + +* `defaultTemplate` +* `templates`.`*name* + +The following flake output attributes must be *Nixpkgs overlays*: + +* `overlay` +* `overlays`.`*name* + +The following flake output attributes must be *NixOS modules*: + +* `nixosModule` +* `nixosModules`.`*name* + +The following flake output attributes must be +[bundlers](./nix3-bundle.md): + +* `bundlers`.`*name* +* `defaultBundler` + +In addition, the `hydraJobs` output is evaluated in the same way as +Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested +attribute set of derivations). Similarly, the +`legacyPackages`.*system* output is evaluated like `nix-env -qa`. + +)"" diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md new file mode 100644 index 000000000..36cb96051 --- /dev/null +++ b/src/nix/flake-clone.md @@ -0,0 +1,18 @@ +R""( + +# Examples + +* Check out the source code of the `dwarffs` flake and build it: + + ```console + # nix flake clone dwarffs --dest dwarffs + # cd dwarffs + # nix build + ``` + +# Description + +This command performs a Git or Mercurial clone of the repository +containing the source code of the flake *flake-url*. + +)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-info.md new file mode 100644 index 000000000..fda3171db --- /dev/null +++ b/src/nix/flake-info.md @@ -0,0 +1,99 @@ +R""( + +# Examples + +* Show what `nixpkgs` resolves to: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs + Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Description: A collection of packages for the Nix package manager + Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source + Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Last modified: 2020-12-23 12:36:12 + ``` + +* Show information about `dwarffs` in JSON format: + + ```console + # nix flake info dwarffs --json | jq . + { + "description": "A filesystem that fetches DWARF debug info from the Internet on demand", + "lastModified": 1597153508, + "locked": { + "lastModified": 1597153508, + "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=", + "owner": "edolstra", + "repo": "dwarffs", + "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "type": "github" + }, + "original": { + "id": "dwarffs", + "type": "indirect" + }, + "originalUrl": "flake:dwarffs", + "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source", + "resolved": { + "owner": "edolstra", + "repo": "dwarffs", + "type": "github" + }, + "resolvedUrl": "github:edolstra/dwarffs", + "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5" + } + ``` + +# Description + +This command shows information about the flake specified by the flake +reference *flake-url*. It resolves the flake reference using the +[flake registry](./nix3-registry.md), fetches it, and prints some meta +data. This includes: + +* `Resolved URL`: If *flake-url* is a flake identifier, then this is + the flake reference that specifies its actual location, looked up in + the flake registry. + +* `Locked URL`: A flake reference that contains a commit or content + hash and thus uniquely identifies a specific flake version. + +* `Description`: A one-line description of the flake, taken from the + `description` field in `flake.nix`. + +* `Path`: The store path containing the source code of the flake. + +* `Revision`: The Git or Mercurial commit hash of the locked flake. + +* `Revisions`: The number of ancestors of the Git or Mercurial commit + of the locked flake. Note that this is not available for `github` + flakes. + +* `Last modified`: For Git or Mercurial flakes, this is the commit + time of the commit of the locked flake; for tarball flakes, it's the + most recent timestamp of any file inside the tarball. + +With `--json`, the output is a JSON object with the following fields: + +* `original` and `originalUrl`: The flake reference specified by the + user (*flake-url*) in attribute set and URL representation. + +* `resolved` and `resolvedUrl`: The resolved flake reference (see + above) in attribute set and URL representation. + +* `locked` and `lockedUrl`: The locked flake reference (see above) in + attribute set and URL representation. + +* `description`: See `Description` above. + +* `path`: See `Path` above. + +* `revision`: See `Revision` above. + +* `revCount`: See `Revisions` above. + +* `lastModified`: See `Last modified` above. + +)"" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md new file mode 100644 index 000000000..c66154ad5 --- /dev/null +++ b/src/nix/flake-init.md @@ -0,0 +1,54 @@ +R""( + +# Examples + +* Create a flake using the default template: + + ```console + # nix flake init + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template: + + ```console + # nix flake init -t templates#simpleContainer + ``` + +# Description + +This command creates a flake in the current directory by copying the +files of a template. It will not overwrite existing files. The default +template is `templates#defaultTemplate`, but this can be overriden +using `-t`. + +# Template definitions + +A flake can declare templates through its `templates` and +`defaultTemplate` output attributes. A template has two attributes: + +* `description`: A one-line description of the template, in CommonMark + syntax. + +* `path`: The path of the directory to be copied. + +Here is an example: + +``` +outputs = { self }: { + + templates.rust = { + path = ./rust; + description = "A simple Rust/Cargo project"; + }; + + templates.defaultTemplate = self.templates.rust; +} +``` + +)"" diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md new file mode 100644 index 000000000..250e13be0 --- /dev/null +++ b/src/nix/flake-list-inputs.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Show the inputs of the `hydra` flake: + + ```console + # nix flake list-inputs github:NixOS/hydra + github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 + ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' + ``` + +# Description + +This command shows the inputs of the flake specified by the flake +referenced *flake-url*. Since it prints the locked inputs that result +from generating or updating the lock file, this command essentially +displays the contents of the flake's lock file in human-readable form. + +)"" diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md new file mode 100644 index 000000000..725695c01 --- /dev/null +++ b/src/nix/flake-new.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Create a flake using the default template in the directory `hello`: + + ```console + # nix flake new hello + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template in the directory `hello`: + + ```console + # nix flake new hello -t templates#trivial + ``` + +# Description + +This command creates a flake in the directory `dest-dir`, which must +not already exist. It's equivalent to: + +```console +# mkdir dest-dir +# cd dest-dir +# nix flake init +``` + +)"" diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md new file mode 100644 index 000000000..1a42c44a0 --- /dev/null +++ b/src/nix/flake-show.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Show the output attributes provided by the `patchelf` flake: + + ```console + github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e + ├───checks + │ ├───aarch64-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux + │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + ├───defaultPackage + │ ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux: package 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b' + ├───hydraJobs + │ ├───build + │ │ ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b' + │ ├───release: derivation 'patchelf-0.12.20201207.f34751b' + │ └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b' + └───overlay: Nixpkgs overlay + ``` + +# Description + +This command shows the output attributes provided by the flake +specified by flake reference *flake-url*. These are the top-level +attributes in the `outputs` of the flake, as well as lower-level +attributes for some standard outputs (e.g. `packages` or `checks`). + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md new file mode 100644 index 000000000..a2ffedd2a --- /dev/null +++ b/src/nix/flake-update.md @@ -0,0 +1,53 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake update --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +* Recreate the lock file (i.e. update all inputs) and commit the new + lock file: + + ```console + # nix flake update --recreate-lock-file --commit-lock-file + … + warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Note that every command that operates on a flake will +also update the lock file if needed, and supports the same +flags. Therefore, + +```console +# nix flake update --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +> **Note** +> +> This command does *not* update locks that are already present unless +> you explicitly ask for it using `--update-input` or +> `--recreate-lock-file`. Thus, if the lock file already has locks for +> every input, then `nix flake update` (without arguments) does +> nothing. + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 066430c5d..2b91faa64 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -104,6 +104,13 @@ struct CmdFlakeUpdate : FlakeCommand return "update flake lock file"; } + std::string doc() override + { + return + #include "flake-update.md" + ; + } + void run(nix::ref store) override { /* Use --refresh by default for 'nix flake update'. */ @@ -134,6 +141,13 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON return "list info about a given flake"; } + std::string doc() override + { + return + #include "flake-info.md" + ; + } + void run(nix::ref store) override { auto flake = getFlake(); @@ -153,6 +167,13 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON return "list flake inputs"; } + std::string doc() override + { + return + #include "flake-list-inputs.md" + ; + } + void run(nix::ref store) override { auto flake = lockFlake(); @@ -211,6 +232,13 @@ struct CmdFlakeCheck : FlakeCommand return "check whether the flake evaluates and run its tests"; } + std::string doc() override + { + return + #include "flake-check.md" + ; + } + void run(nix::ref store) override { settings.readOnlyMode = !build; @@ -631,22 +659,11 @@ struct CmdFlakeInit : CmdFlakeInitCommon return "create a flake in the current directory from a template"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a flake using the default template:", - "nix flake init" - }, - Example{ - "To see available templates:", - "nix flake show templates" - }, - Example{ - "To create a flake from a specific template:", - "nix flake init -t templates#nixos-container" - }, - }; + return + #include "flake-init.md" + ; } CmdFlakeInit() @@ -662,6 +679,13 @@ struct CmdFlakeNew : CmdFlakeInitCommon return "create a flake in the specified directory from a template"; } + std::string doc() override + { + return + #include "flake-new.md" + ; + } + CmdFlakeNew() { expectArgs({ @@ -681,6 +705,13 @@ struct CmdFlakeClone : FlakeCommand return "clone flake repository"; } + std::string doc() override + { + return + #include "flake-clone.md" + ; + } + CmdFlakeClone() { addFlag({ @@ -720,22 +751,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun return "copy a flake and all its inputs to a store"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy the dwarffs flake and its dependencies to a binary cache:", - "nix flake archive --to file:///tmp/my-cache dwarffs" - }, - Example{ - "To fetch the dwarffs flake and its dependencies to the local Nix store:", - "nix flake archive dwarffs" - }, - Example{ - "To print the store paths of the flake sources of NixOps without fetching them:", - "nix flake archive --json --dry-run nixops" - }, - }; + return + #include "flake-archive.md" + ; } void run(nix::ref store) override @@ -797,6 +817,13 @@ struct CmdFlakeShow : FlakeCommand return "show the outputs provided by a flake"; } + std::string doc() override + { + return + #include "flake-show.md" + ; + } + void run(nix::ref store) override { auto state = getEvalState(); @@ -955,6 +982,13 @@ struct CmdFlake : NixMultiCommand return "manage Nix flakes"; } + std::string doc() override + { + return + #include "flake.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/flake.md b/src/nix/flake.md new file mode 100644 index 000000000..440c45dd1 --- /dev/null +++ b/src/nix/flake.md @@ -0,0 +1,566 @@ +R""( + +# Description + +`nix flake` provides subcommands for creating, modifying and querying +*Nix flakes*. Flakes are the unit for packaging Nix code in a +reproducible and discoverable way. They can have dependencies on other +flakes, making it possible to have multi-repository Nix projects. + +A flake is a filesystem tree (typically fetched from a Git repository +or a tarball) that contains a file named `flake.nix` in the root +directory. `flake.nix` specifies some metadata about the flake such as +dependencies (called *inputs*), as well as its *outputs* (the Nix +values such as packages or NixOS modules provided by the flake). + +# Flake references + +Flake references (*flakerefs*) are a way to specify the location of a +flake. These have two different forms: + +* An attribute set representation, e.g. + + ```nix + { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + } + ``` + + The only required attribute is `type`. The supported types are + listed below. + +* A URL-like syntax, e.g. + + ``` + github:NixOS/nixpkgs + ``` + + These are used on the command line as a more convenient alternative + to the attribute set representation. For instance, in the command + + ```console + # nix build github:NixOS/nixpkgs#hello + ``` + + `github:NixOS/nixpkgs` is a flake reference (while `hello` is an + output attribute). They are also allowed in the `inputs` attribute + of a flake, e.g. + + ```nix + inputs.nixpkgs.url = github:NixOS/nixpkgs; + ``` + + is equivalent to + + ```nix + inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }; + ``` + +## Examples + +Here are some examples of flake references in their URL-like representation: + +* `.`: The flake in the current directory. +* `/home/alice/src/patchelf`: A flake in some other directory. +* `nixpkgs`: The `nixpkgs` entry in the flake registry. +* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` + entry in the flake registry, with its Git revision overriden to a + specific value. +* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` + repository on GitHub. +* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the + `nixpkgs` repository. +* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A + specific revision of the `nixpkgs` repository. +* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory + of a GitHub repository. +* `git+https://github.com/NixOS/patchelf`: A Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master`: A specific + branch of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: + A specific branch *and* revision of a Git repository. +* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball + flake. + +## Flake reference attributes + +The following generic flake reference attributes are supported: + +* `dir`: The subdirectory of the flake in which `flake.nix` is + located. This parameter enables having multiple flakes in a + repository or tarball. The default is the root directory of the + flake. + +* `narHash`: The hash of the NAR serialisation (in SRI format) of the + contents of the flake. This is useful for flake types such as + tarballs that lack a unique content identifier such as a Git commit + hash. + +In addition, the following attributes are common to several flake +reference types: + +* `rev`: A Git or Mercurial commit hash. + +* `ref`: A Git or Mercurial branch or tag name. + +Finally, some attribute are typically not specified by the user, but +can occur in *locked* flake references and are available to Nix code: + +* `revCount`: The number of ancestors of the commit `rev`. + +* `lastModified`: The timestamp (in seconds since the Unix epoch) of + the last modification of this version of the flake. For + Git/Mercurial flakes, this is the commit time of commit *rev*, while + for tarball flakes, it's the most recent timestamp of any file + inside the tarball. + +## Types + +Currently the `type` attribute can be one of the following: + +* `path`: arbitrary local directories, or local Git trees. The + required attribute `path` specifies the path of the flake. The URL + form is + + ``` + [path:](\?` (see below), + except that the `dir` parameter is derived automatically. For + example, if `/foo/bar` is a Git repository, then the flake reference + `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`. + + If the directory is not inside a Git repository, then the flake + contents is the entire contents of *path*. + + *path* generally must be an absolute path. However, on the command + line, it can be a relative path (e.g. `.` or `./foo`) which is + interpreted as relative to the current directory. In this case, it + must start with `.` to avoid ambiguity with registry lookups + (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative + path). + +* `git`: Git repositories. The location of the repository is specified + by the attribute `url`. + + They have the URL form + + ``` + git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + ``` + + The `ref` attribute defaults to `master`. + + The `rev` attribute must denote a commit that exists in the branch + or tag specified by the `ref` attribute, since Nix doesn't do a full + clone of the remote repository by default (and the Git protocol + doesn't allow fetching a `rev` without a known `ref`). The default + is the commit currently pointed to by `ref`. + + For example, the following are valid Git flake references: + + * `git+https://example.org/my/repo` + * `git+https://example.org/my/repo?dir=flake1` + * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` + * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` + * `git+file:///home/my-user/some-repo/some-repo` + +* `mercurial`: Mercurial repositories. The URL form is similar to the + `git` type, except that the URL schema must be one of `hg+http`, + `hg+https`, `hg+ssh` or `hg+file`. + +* `tarball`: Tarballs. The location of the tarball is specified by the + attribute `url`. + + In URL form, the schema must be `http://`, `https://` or `file://` + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` + or `.tar.bz2`. + +* `github`: A more efficient way to fetch repositories from + GitHub. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + These are downloaded as tarball archives, rather than + through Git. This is often much faster and uses less disk space + since it doesn't require fetching the entire history of the + repository. On the other hand, it doesn't allow incremental fetching + (but full downloads are often faster than incremental fetches!). + + The URL syntax for `github` flakes is: + + ``` + github:/(/)?(\?)? + ``` + + `` specifies the name of a branch or tag (`ref`), or a + commit hash (`rev`). Note that unlike Git, GitHub allows fetching by + commit hash without specifying a branch or tag. + + Some examples: + + * `github:edolstra/dwarffs` + * `github:edolstra/dwarffs/unstable` + * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + +* `indirect`: Indirections through the flake registry. These have the + form + + ``` + [flake:](/(/rev)?)? + ``` + + These perform a lookup of `` in the flake registry. or + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + +# Flake format + +As an example, here is a simple `flake.nix` that depends on the +Nixpkgs flake and provides a single package (i.e. an installable +derivation): + +```nix +{ + description = "A flake for building Hello World"; + + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + + outputs = { self, nixpkgs }: { + + defaultPackage.x86_64-linux = + # Notice the reference to nixpkgs here. + with import nixpkgs { system = "x86_64-linux"; }; + stdenv.mkDerivation { + name = "hello"; + src = self; + buildPhase = "gcc -o hello ./hello.c"; + installPhase = "mkdir -p $out/bin; install -t $out/bin hello"; + }; + + }; +} +``` + +The following attributes are supported in `flake.nix`: + +* `description`: A short, one-line description of the flake. + +* `inputs`: An attrset specifying the dependencies of the flake + (described below). + +* `outputs`: A function that, given an attribute set containing the + outputs of each of the input flakes keyed by their identifier, + yields the Nix values provided by this flake. Thus, in the example + above, `inputs.nixpkgs` contains the result of the call to the + `outputs` function of the `nixpkgs` flake. + + In addition to the outputs of each input, each input in `inputs` + also contains some metadata about the inputs. These are: + + * `outPath`: The path in the Nix store of the flake's source tree. + + * `rev`: The commit hash of the flake's repository, if applicable. + + * `revCount`: The number of ancestors of the revision `rev`. This is + not available for `github` repositories, since they're fetched as + tarballs rather than as Git repositories. + + * `lastModifiedDate`: The commit time of the revision `rev`, in the + format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`, + this is available for both Git and GitHub repositories, so it's + useful for generating (hopefully) monotonically increasing version + strings. + + * `lastModified`: The commit time of the revision `rev` as an integer + denoting the number of seconds since 1970. + + * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of + the flake's source tree. + + The value returned by the `outputs` function must be an attribute + set. The attributes can have arbitrary values; however, various + `nix` subcommands require specific attributes to have a specific + value (e.g. `packages.x86_64-linux` must be an attribute set of + derivations built for the `x86_64-linux` platform). + +## Flake inputs + +The attribute `inputs` specifies the dependencies of a flake, as an +attrset mapping input names to flake references. For example, the +following specifies a dependency on the `nixpkgs` and `import-cargo` +repositories: + +```nix +# A GitHub repository. +inputs.import-cargo = { + type = "github"; + owner = "edolstra"; + repo = "import-cargo"; +}; + +# An indirection through the flake registry. +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Alternatively, you can use the URL-like syntax: + +```nix +inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.nixpkgs.url = "nixpkgs"; +``` + +Each input is fetched, evaluated and passed to the `outputs` function +as a set of attributes with the same name as the corresponding +input. The special input named `self` refers to the outputs and source +tree of *this* flake. Thus, a typical `outputs` function looks like +this: + +```nix +outputs = { self, nixpkgs, import-cargo }: { + ... outputs ... +}; +``` + +It is also possible to omit an input entirely and *only* list it as +expected function argument to `outputs`. Thus, + +```nix +outputs = { self, nixpkgs }: ...; +``` + +without an `inputs.nixpkgs` attribute is equivalent to + +```nix +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Repositories that don't contain a `flake.nix` can also be used as +inputs, by setting the input's `flake` attribute to `false`: + +```nix +inputs.grcov = { + type = "github"; + owner = "mozilla"; + repo = "grcov"; + flake = false; +}; + +outputs = { self, nixpkgs, grcov }: { + packages.x86_64-linux.grcov = stdenv.mkDerivation { + src = grcov; + ... + }; +}; +``` + +Transitive inputs can be overriden from a `flake.nix` file. For +example, the following overrides the `nixpkgs` input of the `nixops` +input: + +```nix +inputs.nixops.inputs.nixpkgs = { + type = "github"; + owner = "my-org"; + repo = "nixpkgs"; +}; +``` + +It is also possible to "inherit" an input from another input. This is +useful to minimize flake dependencies. For example, the following sets +the `nixpkgs` input of the top-level flake to be equal to the +`nixpkgs` input of the `dwarffs` input of the top-level flake: + +```nix +inputs.nixops.follows = "dwarffs/nixpkgs"; +``` + +The value of the `follows` attribute is a `/`-separated sequence of +input names denoting the path of inputs to be followed from the root +flake. + +Overrides and `follows` can be combined, e.g. + +```nix +inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs"; +``` + +sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs` +input of `dwarffs`. It is worth noting, however, that it is generally +not useful to eliminate transitive `nixpkgs` flake inputs in this +way. Most flakes provide their functionality through Nixpkgs overlays +or NixOS modules, which are composed into the top-level flake's +`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant. + +# Lock files + +Inputs specified in `flake.nix` are typically "unlocked" in the sense +that they don't specify an exact revision. To ensure reproducibility, +Nix will automatically generate and use a *lock file* called +`flake.lock` in the flake's directory. The lock file contains a graph +structure isomorphic to the graph of dependencies of the root +flake. Each node in the graph (except the root node) maps the +(usually) unlocked input specifications in `flake.nix` to locked input +specifications. Each node also contains some metadata, such as the +dependencies (outgoing edges) of the node. + +For example, if `flake.nix` has the inputs in the example above, then +the resulting lock file might be: + +```json +{ + "version": 7, + "root": "n1", + "nodes": { + "n1": { + "inputs": { + "nixpkgs": "n2", + "import-cargo": "n3", + "grcov": "n4" + } + }, + "n2": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013", + "type": "github", + "lastModified": 1580555482, + "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8=" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "n3": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "import-cargo", + "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422", + "type": "github", + "lastModified": 1567183309, + "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc=" + }, + "original": { + "owner": "edolstra", + "repo": "import-cargo", + "type": "github" + } + }, + "n4": { + "inputs": {}, + "locked": { + "owner": "mozilla", + "repo": "grcov", + "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e", + "type": "github", + "lastModified": 1580729070, + "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI=" + }, + "original": { + "owner": "mozilla", + "repo": "grcov", + "type": "github" + }, + "flake": false + } + } +} +``` + +This graph has 4 nodes: the root flake, and its 3 dependencies. The +nodes have arbitrary labels (e.g. `n1`). The label of the root node of +the graph is specified by the `root` attribute. Nodes contain the +following fields: + +* `inputs`: The dependencies of this node, as a mapping from input + names (e.g. `nixpkgs`) to node labels (e.g. `n2`). + +* `original`: The original input specification from `flake.lock`, as a + set of `builtins.fetchTree` arguments. + +* `locked`: The locked input specification, as a set of + `builtins.fetchTree` arguments. Thus, in the example above, when we + build this flake, the input `nixpkgs` is mapped to revision + `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs` + repository on GitHub. + + It also includes the attribute `narHash`, specifying the expected + contents of the tree in the Nix store (as computed by `nix + hash-path`), and may include input-type-specific attributes such as + the `lastModified` or `revCount`. The main reason for these + attributes is to allow flake inputs to be substituted from a binary + cache: `narHash` allows the store path to be computed, while the + other attributes are necessary because they provide information not + stored in the store path. + +* `flake`: A Boolean denoting whether this is a flake or non-flake + dependency. Corresponds to the `flake` attribute in the `inputs` + attribute in `flake.nix`. + +The `original` and `locked` attributes are omitted for the root +node. This is because we cannot record the commit hash or content hash +of the root flake, since modifying `flake.lock` will invalidate these. + +The graph representation of lock files allows circular dependencies +between flakes. For example, here are two flakes that reference each +other: + +```nix +{ + inputs.b = ... location of flake B ...; + # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is + # *this* 'a'. + inputs.b.inputs.a.follows = ""; + outputs = { self, b }: { + foo = 123 + b.bar; + xyzzy = 1000; + }; +} +``` + +and + +```nix +{ + inputs.a = ... location of flake A ...; + inputs.a.inputs.b.follows = ""; + outputs = { self, a }: { + bar = 456 + a.xyzzy; + }; +} +``` + +Lock files transitively lock direct as well as indirect +dependencies. That is, if a lock file exists and is up to date, Nix +will not look at the lock files of dependencies. However, lock file +generation itself *does* use the lock files of dependencies by +default. + +)"" From 1047cb1e53358755cb12a5361cfc99118ed7e159 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 13:21:31 +0100 Subject: [PATCH 255/477] Command: Remove examples() --- doc/manual/generate-manpage.nix | 5 ----- src/libutil/args.cc | 30 ------------------------------ src/libutil/args.hh | 14 -------------- 3 files changed, 49 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 9f30c8fbc..c2c748464 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -20,11 +20,6 @@ let (attrNames def.commands)) + "\n" else "") - + (if def.examples or [] != [] - then - "# Examples\n\n" - + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else "") + (if def ? doc then def.doc + "\n\n" else "") diff --git a/src/libutil/args.cc b/src/libutil/args.cc index a929ea5ac..fb5cb80fb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -353,36 +353,6 @@ void printTable(std::ostream & out, const Table2 & table) } } -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } -} - -nlohmann::json Command::toJSON() -{ - auto exs = nlohmann::json::array(); - - for (auto & example : examples()) { - auto ex = nlohmann::json::object(); - ex["description"] = example.description; - ex["command"] = chomp(stripIndentation(example.command)); - exs.push_back(std::move(ex)); - } - - auto res = Args::toJSON(); - res["examples"] = std::move(exs); - return res; -} - MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 68bbbb4f7..6ed541a32 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -228,25 +228,11 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; - struct Example - { - std::string description; - std::string command; - }; - - typedef std::list Examples; - - virtual Examples examples() { return Examples(); } - typedef int Category; static constexpr Category catDefault = 0; virtual Category category() { return catDefault; } - - void printHelp(const string & programName, std::ostream & out) override; - - nlohmann::json toJSON() override; }; typedef std::map()>> Commands; From 26e502ceb567e48f3eb28fb1848430ab69cb7606 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 16:41:07 +0100 Subject: [PATCH 256/477] Add TODO --- src/nix/bundle.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index c183a170d..5e2298376 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -29,4 +29,8 @@ for more details. > > This command only works on Linux. +# Bundler definitions + +TODO + )"" From 5178211e963fa111f84c4881b22cc506d5254fde Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 18:33:42 +0100 Subject: [PATCH 257/477] Add 'nix' manpage --- src/nix/main.cc | 7 +++ src/nix/nix.md | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 src/nix/nix.md diff --git a/src/nix/main.cc b/src/nix/main.cc index afe7cb8d7..b2406fafe 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -184,6 +184,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { return "a tool for reproducible and declarative configuration management"; } + + std::string doc() override + { + return + #include "nix.md" + ; + } }; static void showHelp(std::vector subcommand) diff --git a/src/nix/nix.md b/src/nix/nix.md new file mode 100644 index 000000000..d10de7c01 --- /dev/null +++ b/src/nix/nix.md @@ -0,0 +1,119 @@ +R""( + +# Examples + +* Create a new flake: + + ```console + # nix flake new hello + # cd hello + ``` + +* Build the flake in the current directory: + + ```console + # nix build + # ./result/bin/hello + Hello, world! + ``` + +* Run the flake in the current directory: + + ```console + # nix run + Hello, world! + ``` + +* Start a development shell for hacking on this flake: + + ```console + # nix develop + # unpackPhase + # cd hello-* + # configurePhase + # buildPhase + # ./hello + Hello, world! + # installPhase + # ../outputs/out/bin/hello + Hello, world! + ``` + +# Description + +Nix is a tool for building software, configurations and other +artifacts in a reproducible and declarative way. For more information, +see the [Nix homepage](https://nixos.org/) or the [Nix +manual](https://nixos.org/manual/nix/stable/). + +# Installables + +Many `nix` subcommands operate on one or more *installables*. These are +command line arguments that represent something that can be built in +the Nix store. Here are the recognised types of installables: + +* **Flake output attributes**: `nixpkgs#hello` + + These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a + flake reference and *attrpath* is an optional attribute path. For + more information on flakes, see [the `nix flake` manual + page](./nix3-flake.md). Flake references are most commonly a flake + identifier in the flake registry (e.g. `nixpkgs`) or a path + (e.g. `/path/to/my-flake` or `.`). + + If *attrpath* is omitted, Nix tries some default values; for most + subcommands, the default is `defaultPackage.`*system* + (e.g. `defaultPackage.x86_64-linux`), but some subcommands have + other defaults. If *attrpath* *is* specified, *attrpath* is + interpreted as relative to one or more prefixes; for most + subcommands, these are `packages.`*system*, + `legacyPackages.*system*` and the empty prefix. Thus, on + `x86_64-linux` `nix build nixpkgs#hello` will try to build the + attributes `packages.x86_64-linux.hello`, + `legacyPackages.x86_64-linux.hello` and `hello`. + +* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + + These are paths inside the Nix store, or symlinks that resolve to a + path in the Nix store. + +* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` + + Store derivations are store paths with extension `.drv` and are a + low-level representation of a build-time dependency graph used + internally by Nix. By default, if you pass a store derivation to a + `nix` subcommand, it will operate on the *output paths* of the + derivation. For example, `nix path-info` prints information about + the output paths: + + ```console + # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv + [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + ``` + + If you want to operate on the store derivation itself, pass the + `--derivation` flag. + +* **Nix attributes**: `--file /path/to/nixpkgs hello` + + When the `-f` / `--file` *path* option is given, installables are + interpreted as attribute paths referencing a value returned by + evaluating the Nix file *path*. + +* **Nix expressions**: `--expr '(import {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. + + When the `--expr` option is given, all installables are interpreted + as Nix expressions. You may need to specify `--impure` if the + expression references impure inputs (such as ``). + +For most commands, if no installable is specified, the default is `.`, +i.e. Nix will operate on the default flake output attribute of the +flake in the current directory. + +# Nix stores + +Most `nix` subcommands operate on a *Nix store*. + +TODO: list store types, options + +)"" From 29bd63e9907cabc5643aaa3f570b9ff5b2d88268 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 19:55:21 +0000 Subject: [PATCH 258/477] Test nix-instantiate with binary cache store Trying to make sure it work with obscurers stores. --- tests/binary-cache.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..8f1c6f14d 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,15 +1,20 @@ source common.sh +# We can produce drvs directly into the binary cache clearStore -clearCache +clearCacheCache +nix-instantiate --store "file://$cacheDir" dependencies.nix # Create the binary cache. +clearStore +clearCache outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to file://$cacheDir $outPath -basicTests() { +basicDownloadTests() { + # No uploading tests bcause upload with force HTTP doesn't work. # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. @@ -44,12 +49,12 @@ basicTests() { # Test LocalBinaryCacheStore. -basicTests +basicDownloadTests # Test HttpBinaryCacheStore. export _NIX_FORCE_HTTP=1 -basicTests +basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. From 57062179ce36e35715284d2ef570f8cb0b90198d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 16:05:09 +0000 Subject: [PATCH 259/477] Move some PKI stuff from LocalStore to Store --- src/libstore/local-store.cc | 9 --------- src/libstore/local-store.hh | 12 ------------ src/libstore/misc.cc | 9 +++++++++ src/libstore/store-api.hh | 13 +++++++++++++ 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..1eb2dec75 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1092,15 +1092,6 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } -const PublicKeys & LocalStore::getPublicKeys() -{ - auto state(_state.lock()); - if (!state->publicKeys) - state->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *state->publicKeys; -} - - void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..d97645058 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,10 +35,6 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const std::string name() override { return "Local Store"; } }; @@ -75,8 +71,6 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); - - std::unique_ptr publicKeys; }; Sync _state; @@ -94,12 +88,6 @@ public: const Path tempRootsDir; const Path fnTempRoots; -private: - - const PublicKeys & getPublicKeys(); - -public: - // Hack for build-remote.cc. PathSet locksHeld; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..0d4190a56 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,4 +282,13 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } +const PublicKeys & Store::getPublicKeys() +{ + auto cryptoState(_cryptoState.lock()); + if (!cryptoState->publicKeys) + cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *cryptoState->publicKeys; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..e3de6db17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,6 +189,10 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + Setting requireSigs{this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -710,11 +714,20 @@ public: return toRealPath(printStorePath(storePath)); } + const PublicKeys & getPublicKeys(); + virtual void createUser(const std::string & userName, uid_t userId) { } protected: + struct CryptoState + { + std::unique_ptr publicKeys; + }; + + Sync _cryptoState; + Stats stats; /* Unsupported methods. */ From 12f7a1f65becfe3b036d0f840ee4a05f2f1f857c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:07:28 +0000 Subject: [PATCH 260/477] build-remote no longer requires local store be local --- src/build-remote/build-remote.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8348d8c91..350bd6cef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -71,11 +71,15 @@ static int main_build_remote(int argc, char * * argv) initPlugins(); - auto store = openStore().cast(); + auto store = openStore(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = store->stateDir + "/current-load"; + currentLoad = "/current-load"; + if (auto localStore = store.dynamic_pointer_cast()) + currentLoad = std::string { localStore->stateDir } + currentLoad; + else + currentLoad = settings.nixStateDir + currentLoad; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; @@ -288,8 +292,9 @@ connected: if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - for (auto & i : missing) - store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ + if (auto localStore = store.dynamic_pointer_cast()) + for (auto & i : missing) + localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } From 450c3500f1e3fb619636c0a29d65300020f99d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:36:52 +0000 Subject: [PATCH 261/477] Crudely make worker only provide a Store, not LocalStore We downcast in a few places, this will be refactored to be better later. --- src/libstore/build/derivation-goal.cc | 66 ++++++++++++++++++--------- src/libstore/build/worker.cc | 6 ++- src/libstore/build/worker.hh | 9 ++-- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 47d11dc53..de32f60db 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -848,14 +848,16 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; + if (auto localStore = dynamic_cast(&worker.store)) { + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } #endif deleteTmpDir(false); @@ -1215,12 +1217,15 @@ void DerivationGoal::startBuilder() useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif + if (auto localStoreP = dynamic_cast(&worker.store)) { + auto & localStore = *localStoreP; + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } } /* Create a temporary directory where the build will take @@ -2182,7 +2187,8 @@ void DerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; + if (auto localStore = dynamic_cast(&worker.store)) + params["root"] = localStore->rootDir; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -3246,7 +3252,13 @@ void DerivationGoal::registerOutputs() } } + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { @@ -3271,8 +3283,8 @@ void DerivationGoal::registerOutputs() /* Since we verified the build, it's now ultimately trusted. */ if (!oldInfo.ultimate) { oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3288,13 +3300,13 @@ void DerivationGoal::registerOutputs() } if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() + localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(newInfo.path); } newInfo.deriver = drvPath; newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); + localStore.signPathInfo(newInfo); finish(newInfo.path); @@ -3302,7 +3314,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({{newInfo.path, newInfo}}); + localStore.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3375,11 +3387,16 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ { + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { infos2.insert_or_assign(newInfo.path, newInfo); } - worker.store.registerValidPaths(infos2); + localStore.registerValidPaths(infos2); } /* In case of a fixed-output derivation hash mismatch, throw an @@ -3577,7 +3594,12 @@ Path DerivationGoal::openLogFile() auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); + Path logDir; + if (auto localStore = dynamic_cast(&worker.store)) + logDir = localStore->logDir; + else + logDir = settings.nixLogDir; + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); createDirs(dir); Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..a9575fb0f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -8,7 +8,7 @@ namespace nix { -Worker::Worker(LocalStore & store) +Worker::Worker(Store & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) @@ -229,7 +229,9 @@ void Worker::run(const Goals & _topGoals) checkInterrupt(); - store.autoGC(false); + // TODO GC interface? + if (auto localStore = dynamic_cast(&store)) + localStore->autoGC(false); /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index bf8cc4586..82e711191 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,9 +2,12 @@ #include "types.hh" #include "lock.hh" -#include "local-store.hh" +#include "store-api.hh" #include "goal.hh" +#include +#include + namespace nix { /* Forward definition. */ @@ -102,7 +105,7 @@ public: /* Set if at least one derivation is not deterministic in check mode. */ bool checkMismatch; - LocalStore & store; + Store & store; std::unique_ptr hook; @@ -124,7 +127,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(LocalStore & store); + Worker(Store & store); ~Worker(); /* Make a goal (with caching). */ From 85f2e9e8fa4f7452a05cfffc901d118a7c861d0a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:54:57 +0000 Subject: [PATCH 262/477] Expose schedule entrypoints to all stores Remote stores still override so the other end schedules. --- src/libstore/binary-cache-store.hh | 7 ------ .../{local-store-build.cc => entry-points.cc} | 6 ++--- src/libstore/dummy-store.cc | 7 ------ src/libstore/local-store.hh | 9 -------- src/libstore/store-api.cc | 23 ------------------- src/libstore/store-api.hh | 6 ++--- 6 files changed, 6 insertions(+), 52 deletions(-) rename src/libstore/build/{local-store-build.cc => entry-points.cc} (91%) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 443a53cac..c2163166c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -108,13 +108,6 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - ref getFSAccessor() override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/entry-points.cc similarity index 91% rename from src/libstore/build/local-store-build.cc rename to src/libstore/build/entry-points.cc index c91cda2fd..9f97d40ba 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/entry-points.cc @@ -5,7 +5,7 @@ namespace nix { -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); @@ -43,7 +43,7 @@ void LocalStore::buildPaths(const std::vector & drvPaths, } } -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, +BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); @@ -63,7 +63,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe } -void LocalStore::ensurePath(const StorePath & path) +void Store::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3c7caf8f2..8f26af685 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -55,13 +55,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - std::optional queryRealisation(const DrvOutput&) override { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d97645058..aa5de31f0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -133,15 +133,6 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void buildPaths( - const std::vector & paths, - BuildMode buildMode) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; - void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..f12a564a1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -747,29 +747,6 @@ const Store::Stats & Store::getStats() } -void Store::buildPaths(const std::vector & paths, BuildMode buildMode) -{ - StorePathSet paths2; - - for (auto & path : paths) { - if (path.path.isDerivation()) { - auto outPaths = queryPartialDerivationOutputMap(path.path); - for (auto & outputName : path.outputs) { - auto currentOutputPathIter = outPaths.find(outputName); - if (currentOutputPathIter == outPaths.end() || - !currentOutputPathIter->second || - !isValidPath(*currentOutputPathIter->second)) - unsupported("buildPaths"); - } - } else - paths2.insert(path.path); - } - - if (queryValidPaths(paths2).size() != paths2.size()) - unsupported("buildPaths"); -} - - void copyStorePath(ref srcStore, ref dstStore, const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3de6db17..4db980fe9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -523,17 +523,17 @@ public: explicitly choosing to allow it). */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) = 0; + BuildMode buildMode = bmNormal); /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(const StorePath & path); /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ virtual void addTempRoot(const StorePath & path) - { unsupported("addTempRoot"); } + { warn("not creating temp root, store doesn't support GC"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed From fed123724679de89d3f56a4c01b5c4c96f93e584 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 19:55:21 +0000 Subject: [PATCH 263/477] Test nix-build with non-local-store --store Just a few small things needed fixing! --- src/libstore/build/derivation-goal.cc | 20 +++++++++++++++++--- tests/binary-cache-build-remote.sh | 13 +++++++++++++ tests/local.mk | 4 +++- 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/binary-cache-build-remote.sh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de32f60db..17f39a86e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -592,9 +592,17 @@ void DerivationGoal::tryToBuild() PathSet lockFiles; /* FIXME: Should lock something like the drv itself so we don't build same CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + if (dynamic_cast(&worker.store)) + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -680,6 +688,12 @@ void DerivationGoal::tryLocalBuild() { /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ + if (!dynamic_cast(&worker.store)) { + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + } unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh new file mode 100644 index 000000000..ed51164a4 --- /dev/null +++ b/tests/binary-cache-build-remote.sh @@ -0,0 +1,13 @@ +source common.sh + +clearStore +clearCacheCache + +# Fails without remote builders +(! nix-build --store "file://$cacheDir" dependencies.nix) + +# Succeeds with default store as build remote. +nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix + +# Succeeds without any build capability because no-op +nix-build --store "file://$cacheDir" -j0 dependencies.nix diff --git a/tests/local.mk b/tests/local.mk index ce94ec80e..aa8b4f9bf 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -9,7 +9,9 @@ nix_tests = \ local-store.sh remote-store.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ - binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ + binary-cache.sh \ + binary-cache-build-remote.sh \ + nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ From d4870462f8f539adeaa6dca476aff6f1f31e1981 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Dec 2020 14:16:06 -0600 Subject: [PATCH 264/477] Cast variants fully for libc++10 libc++10 seems to be stricter on what it allows in variant conversion. I'm not sure what the rules are here, but this is the minimal change needed to get through the compilation errors. --- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/attrs.cc | 2 +- src/libfetchers/github.cc | 4 ++-- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..539ba71f3 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -394,7 +394,7 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; else if (v.type == tPath) - cachedValue = {root->db->setString(getKey(), v.path), v.path}; + cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; else if (v.type == tBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type == tAttrs) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..1360ade39 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -104,7 +104,7 @@ static void fetchTree( else if (attr.value->type == tBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type == tInt) - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 720b19fcd..17fc4041f 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -11,7 +11,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) for (auto & i : json.items()) { if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), i.value().get()); else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..db1ced5d6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -195,14 +195,14 @@ struct GitArchiveInputScheme : InputScheme auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", lastModified); + input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", lastModified} + {"lastModified", uint64_t(lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 07a51059d..0eb401e10 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, - {"revCount", (int64_t) revCount}, + {"revCount", (uint64_t) revCount}, }); if (!_input.getRev()) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 8c0f20475..56c014a8c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -152,7 +152,7 @@ std::pair downloadTarball( } Attrs infoAttrs({ - {"lastModified", lastModified}, + {"lastModified", uint64_t(lastModified)}, {"etag", res.etag}, }); From 9d3aad7b92762973081738a8bc6a562fda45c341 Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:43:22 +0000 Subject: [PATCH 265/477] Update URL where bzip2 can be obtained --- doc/manual/src/installation/prerequisites-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 69b7c5a5e..6825af707 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -30,7 +30,7 @@ have bzip2 installed, including development headers and libraries. If your distribution does not provide these, you can obtain bzip2 from - . + . - `liblzma`, which is provided by XZ Utils. If your distribution does not provide this, you can get it from . From f1e9bda9d1ece9be8c78f5c5345c3adb299bc4aa Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:48:21 +0000 Subject: [PATCH 266/477] Update URL where bzip2 can be obtained --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index daf378997..c1bfc9b53 100644 --- a/configure.ac +++ b/configure.ac @@ -174,9 +174,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"] # Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed From 6262a703636000e525d5c1b877ac28d604a493f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Dec 2020 17:21:19 +0100 Subject: [PATCH 267/477] scanForReferences: Remove misleading comment References have always been determined only by the hash part, not the name or the store prefix. Fixes #4396. --- src/libstore/references.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index eb117b5ba..39c4970c6 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee, TeeSink sink { refsSink, toTee }; std::map backMap; - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ for (auto & i : refs) { auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); From 093de16223b8b93d803e4cd1cc1d3945cb3dfeb1 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 28 Dec 2020 09:30:14 -0800 Subject: [PATCH 268/477] README: fix link to hacking guide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11fe5f932..4686010ef 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From 64904b9d5d32c4201aaf462ae82b736f33785793 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 28 Dec 2020 19:40:04 -0600 Subject: [PATCH 269/477] Fixup --- src/libexpr/eval-cache.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0f84944cd..98d91c905 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -393,9 +393,9 @@ Value & AttrCursor::forceValue() if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == nPath) + else if (v.type() == nPath) cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; - else if (v.type == nBool) + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nAttrs) ; // FIXME: do something? diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e64e3fbb8..ab80be2d3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -103,7 +103,7 @@ static void fetchTree( addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->type == nInt) + else if (attr.value->type() == nInt) attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", From d27eb0ef573b4739967119448779da4a8b2a2cbf Mon Sep 17 00:00:00 2001 From: David McFarland Date: Wed, 30 Dec 2020 16:20:03 -0400 Subject: [PATCH 270/477] Fix insufficent attribute capacity in user profile --- src/nix-env/user-env.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 87387e794..168ac492b 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + DrvInfo::Outputs outputs = i.queryOutputs(true); + StringSet metaNames = i.queryMetaNames(); Value & v(*state.allocValue()); manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); + state.mkAttrs(v, 7 + outputs.size()); mkString(*state.allocAttr(v, state.sType), "derivation"); mkString(*state.allocAttr(v, state.sName), i.queryName()); @@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; @@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); + state.mkAttrs(vMeta, metaNames.size()); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) continue; From e069ddf3258b7eab2074639cade03ba03b0a03a4 Mon Sep 17 00:00:00 2001 From: Sam Lidder Date: Thu, 31 Dec 2020 20:17:37 -0500 Subject: [PATCH 271/477] Fix `configure` error in introduction doc --- doc/manual/src/introduction.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index f01fe7b38..d68445c95 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -165,10 +165,10 @@ You’re then dropped into a shell where you can edit, build and test the package: ```console -[nix-shell]$ tar xf $src +[nix-shell]$ unpackPhase [nix-shell]$ cd pan-* -[nix-shell]$ ./configure -[nix-shell]$ make +[nix-shell]$ configurePhase +[nix-shell]$ buildPhase [nix-shell]$ ./pan/gui/pan ``` From 988dd0a65f562741708f6a7a7a44e333d6a5b205 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Tue, 5 Jan 2021 02:06:25 +0000 Subject: [PATCH 272/477] Fix conversion from JSON to fetch attributes It appears as through the fetch attribute, which is simply a variant with 3 elements, implicitly converts boolean arguments to integers. One must use Explicit to correctly populate it with a boolean. This was missing from the implementation, and resulted in clearly boolean JSON fields being treated as numbers. --- src/libfetchers/attrs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 17fc4041f..a565d19d4 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -15,7 +15,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), Explicit { i.value().get() }); else throw Error("unsupported input attribute type in lock file"); } From 8af4f886e212346afdd1d40789f96f1321da96c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 11:47:29 +0100 Subject: [PATCH 273/477] Fix deadlock in LocalStore::addSignatures() Fixes #4367. --- src/libstore/local-store.cc | 95 +++++++++++++++++++------------------ src/libstore/local-store.hh | 2 + 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..702e7b136 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -736,57 +736,62 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - callback(retrySQLite>([&]() { + callback(retrySQLite>([&]() { auto state(_state.lock()); - - /* Get the path info. */ - auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); - - if (!useQueryPathInfo.next()) - return std::shared_ptr(); - - auto id = useQueryPathInfo.getInt(0); - - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } - - auto info = std::make_shared(path, narHash); - - info->id = id; - - info->registrationTime = useQueryPathInfo.getInt(2); - - auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); - - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); - - info->ultimate = useQueryPathInfo.getInt(5) == 1; - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); - - /* Get the references. */ - auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); - - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - - return info; + return queryPathInfoInternal(*state, path); })); } catch (...) { callback.rethrow(); } } +std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) +{ + /* Get the path info. */ + auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); + + if (!useQueryPathInfo.next()) + return std::shared_ptr(); + + auto id = useQueryPathInfo.getInt(0); + + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } + + auto info = std::make_shared(path, narHash); + + info->id = id; + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + if (s) info->deriver = parseStorePath(s); + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + if (s) info->sigs = tokenizeString(s, " "); + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + if (s) info->ca = parseContentAddressOpt(s); + + /* Get the references. */ + auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); + + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + + return info; +} + + /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { @@ -1608,7 +1613,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); + auto info = std::const_pointer_cast(queryPathInfoInternal(*state, storePath)); info->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..6d29c5960 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,6 +235,8 @@ private: void verifyPath(const Path & path, const StringSet & store, PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); From 146af4ee9bb03968a7322a1ac70dc60c8d5a35e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 16:43:09 +0100 Subject: [PATCH 274/477] Move sodium_init() call --- src/libmain/shared.cc | 9 +++++++++ src/nix-store/nix-store.cc | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2247aeca4..e9f067e35 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,6 +18,10 @@ #include +#if HAVE_SODIUM +#include +#endif + namespace nix { @@ -126,6 +130,11 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif +#if HAVE_SODIUM + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); +#endif + loadConfFile(); startSignalHandlerThread(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54394e921..e1ccece99 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -981,9 +981,6 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string publicKeyFile = *i++; #if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:04:46 +0100 Subject: [PATCH 275/477] crypto.cc: API cleanup and add generate() / to_string() methods --- src/libstore/crypto.cc | 33 ++++++++++++++++++++++++++------- src/libstore/crypto.hh | 24 ++++++++++++++++-------- src/nix-store/nix-store.cc | 17 +++-------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..135ced277 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -8,15 +8,15 @@ namespace nix { -static std::pair split(const string & s) +static std::pair split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,7 +29,12 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) +std::string Key::to_string() const +{ + return name + ":" + base64Encode(key); +} + +SecretKey::SecretKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s) } #endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { #if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; @@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const #endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig, #if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘:’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e1ccece99..e43788bc3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -19,10 +19,6 @@ #include #include -#if HAVE_SODIUM -#include -#endif - namespace nix_store { @@ -980,18 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string secretKeyFile = *i++; string publicKeyFile = *i++; -#if HAVE_SODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); + auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif + writeFile(secretKeyFile, secretKey.to_string()); } From 9374c2baeabe45a22e4b8746dc97f5ce4f030184 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:41:16 +0100 Subject: [PATCH 276/477] Add commands for generating secret/public keys --- src/nix/hash.cc | 5 -- src/nix/key-convert-secret-to-public.md | 19 ++++++ src/nix/key-generate-secret.md | 48 ++++++++++++++ src/nix/sigs.cc | 87 +++++++++++++++++++++++++ tests/binary-cache.sh | 12 ++-- 5 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/nix/key-convert-secret-to-public.md create mode 100644 src/nix/key-generate-secret.md diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 101b67e6a..6fd791f41 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,11 +132,6 @@ struct CmdHash : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..6ff1e1c9b --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign-paths --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 37b8a6712..b2e598ad5 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -141,3 +141,90 @@ struct CmdSignPaths : StorePathsCommand }; static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); + +#if HAVE_SODIUM +struct CmdKeyGenerateSecret : Command +{ + std::optional keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "identifier of the key (e.g. `cache.example.org-1`)", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref(); }}, + {"convert-secret-to-public", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand("key"); +#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..1a06404ed 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -131,14 +131,14 @@ if [ -n "$HAVE_SODIUM" ]; then clearCache clearCacheCache -declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) -publicKey="$(cat $TEST_ROOT/pk1)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 +publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) -res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2)) -badKey="$(cat $TEST_ROOT/pk2)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 +badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) -res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) -otherKey="$(cat $TEST_ROOT/pk3)" +nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 +otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:56:53 +0100 Subject: [PATCH 277/477] Make sodium a required dependency --- Makefile.config.in | 1 - configure.ac | 6 +----- perl/Makefile.config.in | 1 - perl/configure.ac | 6 +----- perl/lib/Nix/Store.xs | 10 ---------- src/libmain/shared.cc | 6 ------ src/libstore/crypto.cc | 29 ----------------------------- src/nix/sigs.cc | 2 -- tests/binary-cache.sh | 4 ---- tests/common.sh.in | 1 - 10 files changed, 2 insertions(+), 64 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 3845b3be0..d1e59e4e7 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -HAVE_SODIUM = @HAVE_SODIUM@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ diff --git a/configure.ac b/configure.ac index c1bfc9b53..2047ed8d2 100644 --- a/configure.ac +++ b/configure.ac @@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA ]) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index c87d4817e..eccfbd9f6 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -2,7 +2,6 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -HAVE_SODIUM = @HAVE_SODIUM@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ diff --git a/perl/configure.ac b/perl/configure.ac index 255744afd..85183c005 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). perlFlags="-I$perllibdir" diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e3b7d389..ad9042a2a 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -14,9 +14,7 @@ #include "util.hh" #include "crypto.hh" -#if HAVE_SODIUM #include -#endif using namespace nix; @@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32) SV * signString(char * secretKey_, char * msg) PPCODE: try { -#if HAVE_SODIUM auto sig = SecretKey(secretKey_).signDetached(msg); XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } @@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg) int checkSignature(SV * publicKey_, SV * sig_, char * msg) CODE: try { -#if HAVE_SODIUM STRLEN publicKeyLen; unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) @@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) throw Error("signature is not valid"); RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e9f067e35..6751a3744 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,9 +18,7 @@ #include -#if HAVE_SODIUM #include -#endif namespace nix { @@ -130,10 +128,8 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif -#if HAVE_SODIUM if (sodium_init() == -1) throw Error("could not initialise libsodium"); -#endif loadConfFile(); @@ -283,9 +279,7 @@ void printVersion(const string & programName) #if HAVE_BOEHMGC cfg.push_back("gc"); #endif -#if HAVE_SODIUM cfg.push_back("signed-caches"); -#endif std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 135ced277..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,9 +2,7 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include -#endif namespace nix { @@ -37,70 +35,46 @@ std::string Key::to_string() const SecretKey::SecretKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); -#endif } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -} -#endif - std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } SecretKey SecretKey::generate(std::string_view name) { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); -#else - noSodium(); -#endif } PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); auto key = publicKeys.find(std::string(ss.first)); @@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index b2e598ad5..14e2c9761 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -142,7 +142,6 @@ struct CmdSignPaths : StorePathsCommand static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); -#if HAVE_SODIUM struct CmdKeyGenerateSecret : Command { std::optional keyName; @@ -227,4 +226,3 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); -#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 1a06404ed..355a37d97 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -125,8 +125,6 @@ grep -q "copying path.*input-0" $TEST_ROOT/log grep -q "copying path.*top" $TEST_ROOT/log -if [ -n "$HAVE_SODIUM" ]; then - # Create a signed binary cache. clearCache clearCacheCache @@ -181,8 +179,6 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" -fi # HAVE_LIBSODIUM - unset _NIX_FORCE_HTTP diff --git a/tests/common.sh.in b/tests/common.sh.in index 5e00d64f1..5489c0c44 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -34,7 +34,6 @@ coreutils=@coreutils@ export dot=@dot@ export SHELL="@bash@" export PAGER=cat -export HAVE_SODIUM="@HAVE_SODIUM@" export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: [PATCH 278/477] Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 232 --------------- src/nix/local.mk | 1 - src/nix/prefetch.cc | 352 +++++++++++++++++++++++ src/nix/store-prefetch-file.md | 32 +++ src/nix/store-prefetch-tarball.md | 31 ++ 6 files changed, 416 insertions(+), 234 deletions(-) delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc create mode 100644 src/nix/prefetch.cc create mode 100644 src/nix/store-prefetch-file.md create mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include - -#include -#include -#include - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional expectedHash; - Hash hash(ht); - std::optional storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md new file mode 100644 index 000000000..535d7e022 --- /dev/null +++ b/src/nix/store-prefetch-tarball.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download a tarball and unpack it, unless it already exists in the + Nix store: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ + --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= + ``` + +# Description + +This command downloads a tarball or zip file from *url*, unpacks it, +and adds the unpacked tree to the Nix store. It prints out the +resulting store path and the NAR hash of that store path. + +The name component of the store path defaults to `source`, but this +can be overriden using `--name`. + +)"" From 93f1678ec60bcacfcc857f361b5f63e37c498eb4 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 01:53:57 +0000 Subject: [PATCH 279/477] Allow Flake inputs to accept boolean and integer attributes I believe that this makes it possible to do things like Git inputs with submodules, but it also likely applies to other input types from libfetchers. --- src/libexpr/flake/flake.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4f021570c..41c93bcaa 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,11 +120,16 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) + if (attr.value->type() == nString) { attrs.emplace(attr.name, attr.value->string.s); - else - throw TypeError("flake input attribute '%s' is %s while a string is expected", + } else if (attr.value->type() == nBool) { + attrs.emplace(attr.name, Explicit{ attr.value->boolean }); + } else if (attr.value->type() == nInt) { + attrs.emplace(attr.name, attr.value->integer); + } else { + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", attr.name, showType(*attr.value)); + } } } catch (Error & e) { e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); From ba0f841a078402f95cf93693c3749743c3ab6246 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 03:13:42 +0000 Subject: [PATCH 280/477] Use switch statement instead of sequence of ifs --- src/libexpr/flake/flake.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 41c93bcaa..9f1e4063f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,15 +120,19 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) { - attrs.emplace(attr.name, attr.value->string.s); - } else if (attr.value->type() == nBool) { - attrs.emplace(attr.name, Explicit{ attr.value->boolean }); - } else if (attr.value->type() == nInt) { - attrs.emplace(attr.name, attr.value->integer); - } else { - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", - attr.name, showType(*attr.value)); + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + attr.name, showType(*attr.value)); } } } catch (Error & e) { From 48a9be2aabf6620ceb00caf7c4c917e4e0a81446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 10:44:55 +0100 Subject: [PATCH 281/477] Remove mkIntFlag --- src/libutil/args.hh | 26 +++++++++++++++----------- src/nix/verify.cc | 10 +++++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6ed541a32..3e84ac64a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -68,8 +68,12 @@ protected: , arity(ArityAny) { } - template - Handler(T * dest) + Handler(std::string * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) : fun([=](std::vector ss) { *dest = ss[0]; }) , arity(1) { } @@ -79,6 +83,15 @@ protected: : fun([=](std::vector ss) { *dest = val; }) , arity(0) { } + + template + Handler(I * dest) + : fun([=](std::vector ss) { + if (!string2Int(ss[0], *dest)) + throw UsageError("'%s' is not an integer", ss[0]); + }) + , arity(1) + { } }; /* Flags. */ @@ -161,15 +174,6 @@ public: }); } - template - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag(shortName, longName, description, [=](I n) { - *dest = n; - }); - } - template void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 16d42349f..620109aac 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,6 +20,7 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + addFlag({ .longName = "substituter", .shortName = 's', @@ -27,7 +28,14 @@ struct CmdVerify : StorePathsCommand .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "require that each path has at least N valid signatures", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override From 1d4954e73e389d319416bf29e949b4b1cbc9ebd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:40:36 +0100 Subject: [PATCH 282/477] Remove mkFlag integer specialisation --- src/libmain/shared.cc | 16 +++++++++++++--- src/libutil/args.hh | 18 ------------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6751a3744..223020378 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -211,9 +211,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); + const std::string & description, const std::string & dest) + { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"n"}, + .handler = {[=](std::string s) { + unsigned int n; + if (!string2Int(s, n)) + throw UsageError("'%s' is not an integer", s); + settings.set(dest, std::to_string(n)); + }} }); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3e84ac64a..c54b0efaf 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -174,24 +174,6 @@ public: }); } - template - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {"N"}, - .handler = {[=](std::string s) { - I n; - if (!string2Int(s, n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); From 29a445840a4f01dfb1533806f8dfc28f7dc4bee9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:42:44 +0100 Subject: [PATCH 283/477] Remove unused mkFlag1 --- src/libutil/args.hh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c54b0efaf..62b9516d8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -143,19 +143,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {label}, - .handler = {[=](std::string s) { fun(s); }} - }); - } - void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:22:21 +0100 Subject: [PATCH 284/477] string2Int(): Return std::optional --- src/libexpr/attr-path.cc | 14 ++++++-------- src/libexpr/get-drvs.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 +++---- src/libstore/build/derivation-goal.cc | 4 +--- src/libstore/globals.cc | 8 ++++++-- src/libstore/local-store.cc | 4 +++- src/libstore/names.cc | 12 ++++++------ src/libstore/nar-info.cc | 8 ++++++-- src/libstore/profiles.cc | 11 +++++------ src/libstore/store-api.cc | 13 +++++++------ src/libutil/args.hh | 4 +++- src/libutil/config.cc | 4 +++- src/libutil/util.hh | 18 ++++++++++++------ src/nix-env/nix-env.cc | 19 +++++++++---------- src/nix/profile.cc | 5 ++--- 17 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2d37dcb7e..9dd557205 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -52,9 +52,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,7 +63,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { if (v->type() != nAttrs) throw TypeError( @@ -82,17 +80,17 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 32c115c12..1a3990ea1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -214,8 +214,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int(v->string.s)) + return *n; } return def; } @@ -228,8 +228,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float(v->string.s)) + return *n; } return def; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index bcb904c0d..d1003de57 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme if (name == "rev" || name == "narHash") input.attrs.insert_or_assign(name, value); else if (name == "revCount" || name == "lastModified") { - uint64_t n; - if (!string2Int(value, n)) + if (auto n = string2Int(value)) + input.attrs.insert_or_assign(name, *n); + else throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input.attrs.insert_or_assign(name, n); } else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 223020378..f1feeddd6 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,10 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - unsigned int n; - if (!string2Int(s, n)) + if (auto n = string2Int(s)) + settings.set(dest, std::to_string(*n)); + else throw UsageError("'%s' is not an integer", s); - settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index ffae5d796..38f627b44 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -70,10 +70,9 @@ template N getIntArg(const string & opt, s.resize(s.size() - 1); } } - N n; - if (!string2Int(s, n)) - throw UsageError("'%1%' requires an integer argument", opt); - return n * multiplier; + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%1%' requires an integer argument", opt); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index af3ab87a9..35f365795 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ad66ef8a8..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -228,8 +228,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702e7b136..c61f34275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int(c1); + auto n2 = string2Int(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..01e2fcc7b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -932,19 +932,20 @@ std::optional decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 62b9516d8..823d843aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,7 +87,9 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (!string2Int(ss[0], *dest)) + if (auto n = string2Int(ss[0])) + *dest = *n; + else throw UsageError("'%s' is not an integer", ss[0]); }) , arity(1) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7af3e7883..7467e5ac0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -230,7 +230,9 @@ template void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); - if (!string2Int(str, value)) + if (auto n = string2Int(str)) + value = *n; + else throw UsageError("setting '%s' has invalid value '%s'", name, str); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0f82bed78..7a4d5fe92 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -397,21 +397,27 @@ bool statusOk(int status); /* Parse a string into an integer. */ -template bool string2Int(const string & s, N & n) +template +std::optional string2Int(const std::string & s) { - if (string(s, 0, 1) == "-" && !std::numeric_limits::is_signed) - return false; + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return {}; std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } /* Parse a string into a float. */ -template bool string2Float(const string & s, N & n) +template +std::optional string2Float(const string & s) { std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c2e075ed..9963f05d9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - GenerationNumber dstGen; - if (!string2Int(opArgs.front(), dstGen)) + if (auto dstGen = string2Int(opArgs.front())) + switchGeneration(globals, *dstGen); + else throw UsageError("expected a generation number"); - - switchGeneration(globals, dstGen); } @@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - GenerationNumber max; - if (!string2Int(str_max, max) || max == 0) + auto max = string2Int(str_max); + if (!max || *max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { - GenerationNumber n; - if (!string2Int(i, n)) + if (auto n = string2Int(i)) + gens.insert(*n); + else throw UsageError("invalid generation number '%1%'", i); - gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d8d2b3a70..8cdd34a20 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,9 +209,8 @@ public: std::vector res; for (auto & s : _matchers) { - size_t n; - if (string2Int(s, n)) - res.push_back(n); + if (auto n = string2Int(s)) + res.push_back(*n); else if (store->isStorePath(s)) res.push_back(s); else From 17beae299d5e6bb511c453d0b9d0d7ef906b3d14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:51:19 +0100 Subject: [PATCH 285/477] Support binary unit prefixes in command line arguments --- src/libmain/shared.cc | 6 ++---- src/libmain/shared.hh | 17 +---------------- src/libutil/args.hh | 5 +---- src/libutil/util.hh | 28 +++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f1feeddd6..e797c2fb9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,8 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - if (auto n = string2Int(s)) - settings.set(dest, std::to_string(*n)); - else - throw UsageError("'%s' is not an integer", s); + auto n = string2IntWithUnitPrefix(s); + settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 38f627b44..edc7b5efa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -57,22 +57,7 @@ template N getIntArg(const string & opt, { ++i; if (i == end) throw UsageError("'%1%' requires an argument", opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%1%' requires an integer argument", opt); + return string2IntWithUnitPrefix(*i); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 823d843aa..3783bc84f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,10 +87,7 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (auto n = string2Int(ss[0])) - *dest = *n; - else - throw UsageError("'%s' is not an integer", ss[0]); + *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7a4d5fe92..ab0bd865a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,12 +401,34 @@ template std::optional string2Int(const std::string & s) { if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return {}; + return std::nullopt; std::istringstream str(s); N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; +} + +/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + 'T' denoting a binary unit prefix. */ +template +N string2IntWithUnitPrefix(std::string s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.resize(s.size() - 1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); } /* Parse a string into a float. */ @@ -417,7 +439,7 @@ std::optional string2Float(const string & s) N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; } From e21aee58f6dd7785df50d5d2a473feb5f6b2ed4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 14:17:06 +0100 Subject: [PATCH 286/477] Fix tests --- src/libutil/tests/tests.cc | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 35a5d27bb..58df9c5ac 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -320,20 +320,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Float, emptyString) { - double n; - ASSERT_EQ(string2Float("", n), false); + ASSERT_EQ(string2Float(""), std::nullopt); } TEST(string2Float, trivialConversions) { - double n; - ASSERT_EQ(string2Float("1.0", n), true); - ASSERT_EQ(n, 1.0); + ASSERT_EQ(string2Float("1.0"), 1.0); - ASSERT_EQ(string2Float("0.0", n), true); - ASSERT_EQ(n, 0.0); + ASSERT_EQ(string2Float("0.0"), 0.0); - ASSERT_EQ(string2Float("-100.25", n), true); - ASSERT_EQ(n, (-100.25)); + ASSERT_EQ(string2Float("-100.25"), -100.25); } /* ---------------------------------------------------------------------------- @@ -341,20 +336,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Int, emptyString) { - double n; - ASSERT_EQ(string2Int("", n), false); + ASSERT_EQ(string2Int(""), std::nullopt); } TEST(string2Int, trivialConversions) { - double n; - ASSERT_EQ(string2Int("1", n), true); - ASSERT_EQ(n, 1); + ASSERT_EQ(string2Int("1"), 1); - ASSERT_EQ(string2Int("0", n), true); - ASSERT_EQ(n, 0); + ASSERT_EQ(string2Int("0"), 0); - ASSERT_EQ(string2Int("-100", n), true); - ASSERT_EQ(n, (-100)); + ASSERT_EQ(string2Int("-100"), -100); } /* ---------------------------------------------------------------------------- From 1db3f84baccc30ac38227c1f7edc3bfbc8e5ff5b Mon Sep 17 00:00:00 2001 From: Danila Date: Fri, 8 Jan 2021 16:12:21 -0800 Subject: [PATCH 287/477] Upcase "Boolean" in Flake attribute type error Co-authored-by: Eelco Dolstra --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9f1e4063f..61aeae543 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", attr.name, showType(*attr.value)); } } From fdcd62eec59485665b919c048874de05235b5971 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Jan 2021 23:20:02 +0100 Subject: [PATCH 288/477] Add 'nix store gc' command --- src/nix/store-gc.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ src/nix/store-gc.md | 21 +++++++++++++++++++++ tests/flakes.sh | 6 +++--- 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/nix/store-gc.cc create mode 100644 src/nix/store-gc.md diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..6e9607d03 --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "stop after freeing `n` bytes of disk space", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/tests/flakes.sh b/tests/flakes.sh index 5aec563ac..2b7bcdd68 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -276,18 +276,18 @@ git -C $flake3Dir commit -m 'Add lockfile' # Test whether registry caching works. nix registry list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp -nix-store --gc +nix store gc nix registry list --flake-registry file://$registry --refresh | grep -q flake3 mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache -nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache +nix store gc # get rid of copies in the store to ensure they get fetched to our git cache _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar mv $flake1Dir $flake1Dir.tmp mv $flake2Dir $flake2Dir.tmp -nix-store --gc +nix store gc _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh mv $flake1Dir.tmp $flake1Dir From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 12:36:39 +0100 Subject: [PATCH 289/477] nix store prefetch-tarball -> nix flake prefetch --- src/nix/flake-prefetch.md | 28 +++++++++++ src/nix/flake.cc | 40 ++++++++++++++++ src/nix/prefetch.cc | 77 +++++++++---------------------- src/nix/store-prefetch-tarball.md | 31 ------------- 4 files changed, 89 insertions(+), 87 deletions(-) create mode 100644 src/nix/flake-prefetch.md delete mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2b91faa64..b73b9cf4e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -960,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand } }; +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + struct CmdFlake : NixMultiCommand { CmdFlake() @@ -973,6 +1012,7 @@ struct CmdFlake : NixMultiCommand {"clone", []() { return make_ref(); }}, {"archive", []() { return make_ref(); }}, {"show", []() { return make_ref(); }}, + {"prefetch", []() { return make_ref(); }}, }) { } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 969299489..ce8c85ecf 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv) static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); -struct CmdStorePrefetch : StoreCommand, MixJSON +struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; - bool unpack; std::optional name; HashType hashType = htSHA256; std::optional expectedHash; - CmdStorePrefetch(bool unpack) - : unpack(unpack) + CmdStorePrefetchFile() { addFlag({ .longName = "name", @@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag({ .longName = "expected-hash", - .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .description = "expected hash of the file", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,14 +274,31 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + expectArg("url", &url); } Category category() override { return catUtility; } + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); @@ -299,54 +314,4 @@ struct CmdStorePrefetch : StoreCommand, MixJSON } }; -struct CmdStorePrefetchFile : CmdStorePrefetch -{ - CmdStorePrefetchFile() - : CmdStorePrefetch(false) - { - name = "source"; - - addFlag({ - .longName = "executable", - .description = "make the resulting file executable", - .handler = {&executable, true}, - }); - } - - std::string description() override - { - return "download a file into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-file.md" - ; - } -}; - static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); - -struct CmdStorePrefetchTarball : CmdStorePrefetch -{ - CmdStorePrefetchTarball() - : CmdStorePrefetch(true) - { - name = "source"; - } - - std::string description() override - { - return "download and unpack a tarball into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-tarball.md" - ; - } -}; - -static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md deleted file mode 100644 index 535d7e022..000000000 --- a/src/nix/store-prefetch-tarball.md +++ /dev/null @@ -1,31 +0,0 @@ -R""( - -# Examples - -* Download a tarball and unpack it: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz - Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' - to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash - 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). - ``` - -* Download a tarball and unpack it, unless it already exists in the - Nix store: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ - --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= - ``` - -# Description - -This command downloads a tarball or zip file from *url*, unpacks it, -and adds the unpacked tree to the Nix store. It prints out the -resulting store path and the NAR hash of that store path. - -The name component of the store path defaults to `source`, but this -can be overriden using `--name`. - -)"" From 77c9ceda4be8dd304b07f654d8c059a01d839108 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:42:24 +0100 Subject: [PATCH 290/477] Tweak --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 827adbd05..361c20cc9 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -226,7 +226,7 @@ control what gets deleted and in what order: or TiB units. The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` variables in the Nix +`keep-outputs` and `keep-derivations` settings in the Nix configuration file. By default, the collector prints the total number of freed bytes when it From 6254b1f5d298ff73127d7b0f0da48f142bdc753c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:46:17 +0100 Subject: [PATCH 291/477] Add 'nix store delete' command --- src/nix/store-delete.cc | 45 +++++++++++++++++++++++++++++++++++++++ src/nix/store-delete.md | 24 +++++++++++++++++++++ tests/multiple-outputs.sh | 2 +- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/nix/store-delete.cc create mode 100644 src/nix/store-delete.md diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..f3677763c --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,45 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "do not check whether the paths are reachable from a root", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 7a6ec181d..de573d4fa 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou # Delete one of the outputs and rebuild it. This will cause a hash # rewrite. -nix-store --delete $TEST_ROOT/result-second --ignore-liveness +nix store delete $TEST_ROOT/result-second --ignore-liveness nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result [ "$(cat $TEST_ROOT/result-second/file)" = "second" ] [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Tue, 12 Jan 2021 01:28:00 +0100 Subject: [PATCH 292/477] Don't let 'preferLocalBuild' override 'max-jobs=0' This resolves #3810 by changing the behavior of `max-jobs = 0`, so that specifying the option also avoids local building of derivations with the attribute `preferLocalBuild = true`. --- src/libstore/parsed-derivations.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 11 Jan 2021 22:05:32 -0600 Subject: [PATCH 293/477] Set kern.curproc_arch_affinity=0 to escape Rosetta By default, once you enter x86_64 Rosetta 2, macOS will try to run everything in x86_64. So an x86_64 Nix will still try to use x86_64 even when system = aarch64-darwin. To avoid this we can set kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0, we ignore this preference. This is based on how https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html works. Completely undocumented, but seems to work! Note, you can verify this works with this impure Nix expression: ``` { a = derivation { name = "a"; system = "aarch64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = arm64 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; b = derivation { name = "b"; system = "x86_64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; } ``` --- src/libstore/build/derivation-goal.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..a02ddb950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,6 +52,7 @@ #if __APPLE__ #include +#include #endif #include @@ -2869,6 +2870,10 @@ void DerivationGoal::runChild() throw SysError("failed to initialize builder"); if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } else if (drv->platform == "x86_64-darwin") { From 29007f8bc6ea42ae1f8311f00c0b5e14f04ec9e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 19:57:05 +0100 Subject: [PATCH 294/477] nix profile info -> nix profile list --- src/nix/{profile-info.md => profile-list.md} | 2 +- src/nix/profile.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/nix/{profile-info.md => profile-list.md} (98%) diff --git a/src/nix/profile-info.md b/src/nix/profile-list.md similarity index 98% rename from src/nix/profile-info.md rename to src/nix/profile-list.md index a0c04fc8c..5c29c0b02 100644 --- a/src/nix/profile-info.md +++ b/src/nix/profile-list.md @@ -5,7 +5,7 @@ R""( * Show what packages are installed in the default profile: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cdd34a20..ac60d336c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { @@ -346,7 +346,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro std::string doc() override { return - #include "profile-info.md" + #include "profile-list.md" ; } @@ -408,7 +408,7 @@ struct CmdProfile : NixMultiCommand {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, + {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, }) { } From 2f463e90ed077e066455a9ef6e024b18fd61c4de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 23:51:07 +0100 Subject: [PATCH 295/477] Add 'nix profile history' command Replaces 'nix-env --list-generations'. Similar to 'nix profile diff-closures' but shows only the changes in top-level packages. --- src/nix/command.hh | 2 + src/nix/profile-history.md | 26 +++++++++ src/nix/profile.cc | 114 +++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/nix/profile-history.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 6882db195..3aae57edd 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -261,6 +261,8 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); +std::string showVersions(const std::set & versions); + void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..d0fe40c82 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 -> 509: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 -> 510: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ac60d336c..ca95817d0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include "../nix-env/user-env.hh" #include "profiles.hh" +#include "names.hh" #include #include @@ -21,6 +22,13 @@ struct ProfileElementSource FlakeRef resolvedRef; std::string attrPath; // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } }; struct ProfileElement @@ -29,6 +37,29 @@ struct ProfileElement std::optional source; bool active = true; // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } }; struct ProfileManifest @@ -142,6 +173,46 @@ struct ProfileManifest return std::move(info.path); } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } }; struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile @@ -401,6 +472,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile } }; +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + if (prevGen) + std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); + else + std::cout << fmt("Version %d:\n", gen.number); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -410,6 +523,7 @@ struct CmdProfile : NixMultiCommand {"upgrade", []() { return make_ref(); }}, {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, + {"history", []() { return make_ref(); }}, }) { } From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 13 Jan 2021 10:13:51 +0100 Subject: [PATCH 296/477] Remove a redundant condition in DerivationGoal::tryLocalBuild() --- src/libstore/build/derivation-goal.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..415a55d37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 14:18:04 +0100 Subject: [PATCH 297/477] Convert option descriptions to Markdown --- src/libexpr/common-eval-args.cc | 10 +++++----- src/libmain/common-args.cc | 13 ++++++------- src/libmain/common-args.hh | 4 ++-- src/libmain/shared.cc | 20 ++++++++++---------- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 6 +++--- src/nix/bundle.cc | 6 +++--- src/nix/command.cc | 14 +++++++------- src/nix/copy.cc | 8 ++++---- src/nix/develop.cc | 16 ++++++++-------- src/nix/eval.cc | 6 +++--- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 19 +++++++++---------- src/nix/installables.cc | 26 +++++++++++++------------- src/nix/ls.cc | 6 +++--- src/nix/main.cc | 14 +++++++------- src/nix/path-info.cc | 8 ++++---- src/nix/prefetch.cc | 8 +++++--- src/nix/run.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 6 +++--- src/nix/store-delete.cc | 2 +- src/nix/store-gc.cc | 2 +- src/nix/upgrade-nix.cc | 4 ++-- src/nix/verify.cc | 8 ++++---- src/nix/why-depends.cc | 2 +- 26 files changed, 111 insertions(+), 111 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..ffe782454 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs() { addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up `<...>` file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); addFlag({ .longName = "impure", - .description = "allow access to mutable paths and repositories", + .description = "Allow access to mutable paths and repositories.", .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", - .description = "override a flake registry value", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3e4e475e5..bd5573e5d 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "verbose", .shortName = 'v', - .description = "increase verbosity level", + .description = "Increase the logging verbosity level.", .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", - .description = "decrease verbosity level", + .description = "Decrease the logging verbosity level.", .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", - .description = "enable debug output", + .description = "Set the logging verbosity level to 'debug'.", .handler = {[]() { verbosity = lvlDebug; }}, }); addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding `nix.conf`)", + .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { @@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "format of log output; `raw`, `internal-json`, `bar` " - "or `bar-with-logs`", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "max-jobs", .shortName = 'j', - .description = "maximum number of parallel builds", + .description = "The maximum number of parallel builds.", .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index a4de3dccf..47f341619 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -16,7 +16,7 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); } }; @@ -26,7 +26,7 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "produce JSON output", &json); + mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); } }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e797c2fb9..7e27e95c2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -186,27 +186,27 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-build-output", .shortName = 'Q', - .description = "do not show build output", + .description = "Do not show build output.", .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ .longName = "keep-failed", .shortName ='K', - .description = "keep temporary directories of failed builds", + .description = "Keep temporary directories of failed builds.", .handler = {&(bool&) settings.keepFailed, true}, }); addFlag({ .longName = "keep-going", .shortName ='k', - .description = "keep going after a build fails", + .description = "Keep going after a build fails.", .handler = {&(bool&) settings.keepGoing, true}, }); addFlag({ .longName = "fallback", - .description = "build from source if substitution fails", + .description = "Build from source if substitution fails.", .handler = {&(bool&) settings.tryFallback, true}, }); @@ -225,19 +225,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); }; - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores"); + intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); + intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "do not write to the Nix store", + mkFlag(0, "readonly-mode", "Do not write to the Nix store.", &settings.readOnlyMode); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", &gcWarning, false); addFlag({ .longName = "store", - .description = "URI of the Nix store to use", + .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, .handler = {&(std::string&) settings.storeUri}, }); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ea4bbbab9..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); diff --git a/src/nix/build.cc b/src/nix/build.cc index c2974d983..4cb8ade08 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); addFlag({ .longName = "rebuild", - .description = "rebuild an already built package and compare the result to the existing store paths", + .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, }); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5f558b01e..1789e4598 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand { addFlag({ .longName = "bundler", - .description = "use custom bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { @@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -90,7 +90,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/command.cc b/src/nix/command.cc index 596217775..ba58c7d6b 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) if (recursive) addFlag({ .longName = "no-recursive", - .description = "apply operation to specified paths only", + .description = "Apply operation to specified paths only.", .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', - .description = "apply operation to closure of the specified paths", + .description = "Apply operation to closure of the specified paths.", .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "apply operation to the entire store", &all); + mkFlag(0, "all", "Apply the operation to every store path.", &all); } void StorePathsCommand::run(ref store) @@ -133,7 +133,7 @@ MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "profile to update", + .description = "The profile to update.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "ignore-environment", .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", + .description = "Clear the entire environment (except those specified with `--keep`).", .handler = {&ignoreEnvironment, true}, }); addFlag({ .longName = "keep", .shortName = 'k', - .description = "keep specified environment variable", + .description = "Keep the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { keep.insert(s); }}, }); @@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "unset", .shortName = 'u', - .description = "unset specified environment variable", + .description = "Unset the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { unset.insert(s); }}, }); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2394eb46d..f15031a45 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index edd87f246..578258394 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile { addFlag({ .longName = "redirect", - .description = "redirect a store path to a mutable location", + .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); @@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed instead of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "phase", - .description = "phase to run (e.g. `build` or `configure`)", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", .labels = {"phase-name"}, .handler = {&phase}, }); addFlag({ .longName = "configure", - .description = "run the configure phase", + .description = "Run the `configure` phase.", .handler = {&phase, {"configure"}}, }); addFlag({ .longName = "build", - .description = "run the build phase", + .description = "Run the `build` phase.", .handler = {&phase, {"build"}}, }); addFlag({ .longName = "check", - .description = "run the check phase", + .description = "Run the `check` phase.", .handler = {&phase, {"check"}}, }); addFlag({ .longName = "install", - .description = "run the install phase", + .description = "Run the `install` phase.", .handler = {&phase, {"install"}}, }); addFlag({ .longName = "installcheck", - .description = "run the installcheck phase", + .description = "Run the `installcheck` phase.", .handler = {&phase, {"installCheck"}}, }); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 321df7495..b5049ac65 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); addFlag({ .longName = "apply", - .description = "apply a function to each argument", + .description = "Apply the function *expr* to each argument.", .labels = {"expr"}, .handler = {&apply}, }); addFlag({ .longName = "write-to", - .description = "write a string or attrset of strings to 'path'", + .description = "Write a string or attrset of strings to *path*.", .labels = {"path"}, .handler = {&writeTo}, }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b73b9cf4e..4cd7d77a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -222,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand { addFlag({ .longName = "no-build", - .description = "do not build checks", + .description = "Do not build checks.", .handler = {&build, false} }); } @@ -573,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand addFlag({ .longName = "template", .shortName = 't', - .description = "the template to use", + .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, .completer = {[&](size_t, std::string_view prefix) { @@ -717,7 +717,7 @@ struct CmdFlakeClone : FlakeCommand addFlag({ .longName = "dest", .shortName = 'f', - .description = "destination path", + .description = "Clone the flake to path *dest*.", .labels = {"path"}, .handler = {&destDir} }); @@ -807,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand { addFlag({ .longName = "legacy", - .description = "show the contents of the 'legacyPackages' output", + .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6fd791f41..79d506ace 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,15 +19,15 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); + mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); + mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); + mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") - .description("compute hash modulo specified string") + .description("Compute the hash modulo specified the string.") .labels({"modulus"}) .dest(&modulus); #endif @@ -40,15 +40,14 @@ struct CmdHashBase : Command std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; - break; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } void run() override diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3506c3fcc..50e3b29c4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions() { addFlag({ .longName = "recreate-lock-file", - .description = "recreate lock file from scratch", + .description = "Recreate the flake's lock file from scratch.", .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", - .description = "do not allow any updates to the lock file", + .description = "Do not allow any updates to the flake's lock file.", .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", - .description = "do not write the newly generated lock file", + .description = "Do not write the flake's newly generated lock file.", .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", - .description = "don't use flake registries", + .description = "Don't allow lookups in the flake registries.", .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", - .description = "commit changes to the lock file", + .description = "Commit changes to the flake's lock file.", .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", - .description = "update a specific flake input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", - .description = "use the inputs of the specified flake as registry entries", + .description = "Use the inputs of the specified flake as registry entries.", .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "evaluate *file* rather than the default", + .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", .labels = {"file"}, .handler = {&file}, .completer = completePath }); addFlag({ - .longName ="expr", - .description = "evaluate attributes from *expr*", + .longName = "expr", + .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", .labels = {"expr"}, .handler = {&expr} }); addFlag({ - .longName ="derivation", - .description = "operate on the store derivation rather than its outputs", + .longName = "derivation", + .description = "Operate on the store derivation rather than its outputs.", .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d48287f27..c0b1ecb32 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); + mkFlag('l', "long", "Show detailed file information.", &verbose); + mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); } void listText(ref accessor) diff --git a/src/nix/main.cc b/src/nix/main.cc index b2406fafe..803453dd5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -69,15 +69,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", + .description = "Show usage information.", .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ .longName = "help-config", - .description = "show configuration options", + .description = "Show configuration settings.", .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; + std::cout << "The following configuration settings are available:\n\n"; Table2 tbl; std::map settings; globalConfig.getSettings(settings); @@ -91,25 +91,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", + .description = "Show version information.", .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 30b6a50f8..0fa88f1bf 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); + mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); + mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); + mkFlag(0, "sigs", "Show signatures.", &showSigs); } std::string description() override diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce8c85ecf..a831dcd15 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { addFlag({ .longName = "name", - .description = "store path name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, .handler = {&name} }); addFlag({ .longName = "expected-hash", - .description = "expected hash of the file", + .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON addFlag({ .longName = "executable", - .description = "make the resulting file executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", .handler = {&executable, true}, }); diff --git a/src/nix/run.cc b/src/nix/run.cc index 1340dd46f..ec9388234 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 13f2c8e69..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 14e2c9761..4b6ead6c7 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -101,7 +101,7 @@ struct CmdSignPaths : StorePathsCommand addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath @@ -150,7 +150,7 @@ struct CmdKeyGenerateSecret : Command { addFlag({ .longName = "key-name", - .description = "identifier of the key (e.g. `cache.example.org-1`)", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", .labels = {"name"}, .handler = {&keyName}, }); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index f3677763c..9c8fef191 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -13,7 +13,7 @@ struct CmdStoreDelete : StorePathsCommand { addFlag({ .longName = "ignore-liveness", - .description = "do not check whether the paths are reachable from a root", + .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true} }); } diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 6e9607d03..a2d74066e 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -13,7 +13,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun { addFlag({ .longName = "max", - .description = "stop after freeing `n` bytes of disk space", + .description = "Stop after freeing *n* bytes of disk space.", .labels = {"n"}, .handler = {&options.maxFreed} }); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 79be31e73..299ea40aa 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 620109aac..b2963cf74 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,13 +18,13 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); + mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); @@ -32,7 +32,7 @@ struct CmdVerify : StorePathsCommand addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "require that each path has at least N valid signatures", + .description = "Require that each path has at least *n* valid signatures.", .labels = {"n"}, .handler = {&sigsNeeded} }); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 297b638cc..7a4ca5172 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:27:39 +0100 Subject: [PATCH 298/477] Add 'nix store repair' command --- src/libstore/local-store.hh | 4 +--- src/libstore/store-api.hh | 5 +++++ src/nix-store/nix-store.cc | 2 +- src/nix/store-delete.cc | 1 - src/nix/store-repair.cc | 27 +++++++++++++++++++++++++++ src/nix/store-repair.md | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/nix/store-repair.cc create mode 100644 src/nix/store-repair.md diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d29c5960..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..d1b83933a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -604,6 +604,11 @@ public: virtual ref getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e43788bc3..b97f684a4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -757,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); + store->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 9c8fef191..10245978e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,6 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector storePaths) override { - for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" From d33eca8539d2e66759f7b52fa7b0db4a6a1ba673 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:31:18 +0100 Subject: [PATCH 299/477] Rename 'nix store sign-paths' to 'nix store sign' --- doc/manual/src/advanced-topics/post-build-hook.md | 4 ++-- src/nix/key-generate-secret.md | 2 +- src/nix/main.cc | 2 +- src/nix/sigs.cc | 6 +++--- tests/signing.sh | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index bbdabed41..fcb52d878 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -53,7 +53,7 @@ set -f # disable globbing export IFS=' ' echo "Signing paths" $OUT_PATHS -nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS exec nix copy --to 's3://example-nix-cache' $OUT_PATHS ``` @@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix -> sign-paths`. Nix guarantees the paths will not contain any spaces, +> store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md index 6ff1e1c9b..4938f637c 100644 --- a/src/nix/key-generate-secret.md +++ b/src/nix/key-generate-secret.md @@ -12,7 +12,7 @@ R""( ```console # nix build nixpkgs#hello - # nix store sign-paths --key-file ./secret-key --recursive ./result + # nix store sign --key-file ./secret-key --recursive ./result ``` Finally, we can verify the store paths using the corresponding diff --git a/src/nix/main.cc b/src/nix/main.cc index 803453dd5..398526020 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -129,7 +129,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"make-content-addressable", {"store", "make-content-addressable"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign-paths"}}, + {"sign-paths", {"store", "sign"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 4b6ead6c7..3445182f2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,11 +92,11 @@ struct CmdCopySigs : StorePathsCommand static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", @@ -140,7 +140,7 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); +static auto rCmdSign = registerCommand2({"store", "sign"}); struct CmdKeyGenerateSecret : Command { diff --git a/tests/signing.sh b/tests/signing.sh index bd6280cc6..6aafbeb91 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -47,8 +47,8 @@ expect 2 nix store verify -r $outPath2 --sigs-needed 1 expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix store sign-paths". -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign". +nix store sign --key-file $TEST_ROOT/sk1 $outPath2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 @@ -63,7 +63,7 @@ nix store verify $outPathCA nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store sign --key-file $TEST_ROOT/sk1 $outPathCA nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] From 28ef6ebf914792f9e543e9778248f06e716a859d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:51:27 +0100 Subject: [PATCH 300/477] Typo --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index fc6195cf0..9227369b8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -318,7 +318,7 @@ static int main_nix_daemon(int argc, char * * argv) FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limitting access to thoses is explicitly + standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } From 7a472a76d4dcbbd0eb7832c0bdcb120d32881e8b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jan 2021 00:05:04 +0100 Subject: [PATCH 301/477] Add 'nix daemon' command --- src/nix/command.hh | 2 + .../nix-daemon.cc => nix/daemon.cc} | 114 +++++++++++------- src/nix/daemon.md | 21 ++++ src/nix/main.cc | 3 + tests/common.sh.in | 2 +- 5 files changed, 98 insertions(+), 44 deletions(-) rename src/{nix-daemon/nix-daemon.cc => nix/daemon.cc} (77%) create mode 100644 src/nix/daemon.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 3aae57edd..f325cd906 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -13,6 +13,8 @@ namespace nix { extern std::string programPath; +extern char * * savedArgv; + class EvalState; struct Pos; class Store; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc similarity index 77% rename from src/nix-daemon/nix-daemon.cc rename to src/nix/daemon.cc index 9227369b8..204d4ce6b 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,4 @@ +#include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" @@ -150,7 +151,7 @@ static ref openUncachedStore() } -static void daemonLoop(char * * argv) +static void daemonLoop() { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -232,9 +233,9 @@ static void daemonLoop(char * * argv) setSigChldAction(false); // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && argv[1]) { + if (peer.pidKnown && savedArgv[1]) { string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); } // Handle the connection. @@ -264,6 +265,48 @@ static void daemonLoop(char * * argv) } } +static void runDaemon(bool stdio) +{ + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + } + } else + daemonLoop(); +} static int main_nix_daemon(int argc, char * * argv) { @@ -285,49 +328,34 @@ static int main_nix_daemon(int argc, char * * argv) initPlugins(); - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } - } else { - daemonLoop(argv); - } + runDaemon(stdio); return 0; } } static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index 398526020..418396280 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -52,6 +52,7 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { @@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand("help"); void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { diff --git a/tests/common.sh.in b/tests/common.sh.in index 5489c0c44..e3bcab507 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -73,7 +73,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix-daemon & + nix daemon & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 From 86a2ceeb986609488be1c6794a8e416df3b90c7b Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Wed, 13 Jan 2021 16:56:23 -0800 Subject: [PATCH 302/477] Fix gcc10 build --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 61aeae543..0786fef3d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -128,7 +128,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, Explicit { attr.value->boolean }); break; case nInt: - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, (long unsigned int)attr.value->integer); break; default: throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", From 7af743470c09b835f910d2e25786c080ccfe52c1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 15 Jan 2021 16:37:41 +0000 Subject: [PATCH 303/477] Make public keys and `requireSigs` local-store specific again Thanks @regnat and @edolstra for catching this and comming up with the solution. They way I had generalized those is wrong, because local settings for non-local stores is confusing default. And due to the nature of C++ inheritance, fixing the defaults is more annoying than it should be. Additionally, I thought we might just drop the check in the substitution logic since `Store::addToStore` is now streaming, but @regnat rightfully pointed out that as it downloads dependencies first, that would still be too late, and also waste effort on possibly unneeded/unwanted dependencies. The simple and correct thing to do is just make a store method for the boolean logic, keeping all the setting and key stuff the way it was before. That new method is both used by `LocalStore::addToStore` and the substitution goal check. Perhaps we might eventually make it fancier, e.g. sending the ValidPathInfo to remote stores for them to validate, but this is good enough for now. --- src/libstore/build/substitution-goal.cc | 4 +--- src/libstore/local-store.cc | 14 ++++++++++++- src/libstore/local-store.hh | 14 +++++++++++++ src/libstore/misc.cc | 9 -------- src/libstore/store-api.hh | 28 +++++++++++++------------ 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..f3c9040bc 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -142,9 +142,7 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) + if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) { logWarning({ .name = "Invalid path signature", diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4f48522c6..d6d74a0b0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1098,11 +1098,23 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } } +const PublicKeys & LocalStore::getPublicKeys() +{ + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *state->publicKeys; +} + +bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +{ + return requireSigs && !info.checkSignatures(*this, getPublicKeys()); +} void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + if (checkSigs && pathInfoIsTrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69704d266..9d235ba0a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,6 +35,10 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + Setting requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + const std::string name() override { return "Local Store"; } }; @@ -71,6 +75,8 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); + + std::unique_ptr publicKeys; }; Sync _state; @@ -88,6 +94,12 @@ public: const Path tempRootsDir; const Path fnTempRoots; +private: + + const PublicKeys & getPublicKeys(); + +public: + // Hack for build-remote.cc. PathSet locksHeld; @@ -124,6 +136,8 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + bool pathInfoIsTrusted(const ValidPathInfo &) override; + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0d4190a56..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,13 +282,4 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } -const PublicKeys & Store::getPublicKeys() -{ - auto cryptoState(_cryptoState.lock()); - if (!cryptoState->publicKeys) - cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *cryptoState->publicKeys; -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e6a14afc3..3221cf249 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,10 +189,6 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -376,6 +372,21 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /* Check whether the given valid path info is sufficiently well-formed + (e.g. hash content-address or signature) in order to be included in the + given store. + + These same checks would be performed in addToStore, but this allows an + earlier failure in the case where dependencies need to be added too, but + the addToStore wouldn't fail until those dependencies are added. Also, + we don't really want to add the dependencies listed in a nar info we + don't trust anyyways. + */ + virtual bool pathInfoIsTrusted(const ValidPathInfo &) + { + return true; + } + protected: virtual void queryPathInfoUncached(const StorePath & path, @@ -719,20 +730,11 @@ public: return toRealPath(printStorePath(storePath)); } - const PublicKeys & getPublicKeys(); - virtual void createUser(const std::string & userName, uid_t userId) { } protected: - struct CryptoState - { - std::unique_ptr publicKeys; - }; - - Sync _cryptoState; - Stats stats; /* Unsupported methods. */ From 1e13c79a9165e99be9fccfec8e442d14bb66aef0 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 16 Jan 2021 19:11:10 +0000 Subject: [PATCH 304/477] Document expected output of 'nix store ping'. While interpreting the output is fairly intuitive it would be better to explicitly specify what a good invocation looks like. That this isn't completely obvious (or at least causes folks to second-guess themselves) can be seen in a couple user threads: - https://discourse.nixos.org/t/nixos-cache-fetching-issue/3575/11 - https://discourse.nixos.org/t/newbie-question-cant-get-trivial-example-of-nixops-to-work-on-my-mac/1125/8 --- src/nix/ping-store.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 322093091..79b108d9c 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,4 +27,6 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. +When the command succeeds a zero exit code is returned with no output. + )"" From 1acbb61696c118712417bcd1c59021cc84650e16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 17 Jan 2021 19:49:28 +0100 Subject: [PATCH 305/477] Tweak --- src/nix/ping-store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 79b108d9c..8c846791b 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,6 +27,7 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. -When the command succeeds a zero exit code is returned with no output. +If the command succeeds, Nix returns a exit code of 0 and does not +print any output. )"" From 9432c170e736a6b506d9b35ced5eccff6422ec50 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 17:12:58 +0100 Subject: [PATCH 306/477] Fix the drv output map for non ca derivations With the `ca-derivation` experimental features, non-ca derivations used to have their output paths returned as unknown as long as they weren't built (because of a mistake in the code that systematically erased the previous value) --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c61f34275..ab78f1435 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -914,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); else - outputs.insert_or_assign(outputName, std::nullopt); + outputs.insert({outputName, std::nullopt}); } return outputs; From 11b63740e377202e237b7bc74806b82a1eb8ce11 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 21:26:29 +0100 Subject: [PATCH 307/477] Fix content-addressed flake outputs Prevent some `nix flake` commands to crash by trying to parse a placeholder output as a store path --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 50e3b29c4..34ee238bf 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -501,7 +501,7 @@ std::tuple InstallableF auto drvInfo = DerivationInfo{ std::move(drvPath), - state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), attr->getAttr(state->sOutputName)->getString() }; From ea756b3654931f23839aee9f461a8c891c6ffe43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 14:38:31 +0100 Subject: [PATCH 308/477] --refresh: Imply setting .narinfo disk cache TTL to 0 --- src/libstore/nar-info-disk-cache.cc | 6 ++++-- src/nix/main.cc | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..1d8d2d57e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -109,8 +109,10 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) + // Use a minimum TTL to prevent --refresh from + // nuking the entire disk cache. + (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U)) + (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); diff --git a/src/nix/main.cc b/src/nix/main.cc index 418396280..80422bd24 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -330,8 +330,11 @@ void mainWrapped(int argc, char * * argv) fileTransferSettings.connectTimeout = 1; } - if (args.refresh) + if (args.refresh) { settings.tarballTtl = 0; + settings.ttlNegativeNarInfoCache = 0; + settings.ttlPositiveNarInfoCache = 0; + } args.command->second->prepare(); args.command->second->run(); From 555940f0659e95de7f890ede48e2faba096b3d6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 22:50:39 +0100 Subject: [PATCH 309/477] Use enumerate() --- src/nix/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 4cb8ade08..45f63bb7e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (size_t i = 0; i < buildables.size(); ++i) + for (const auto & [i, buildable] : enumerate(buildables)) { std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; @@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildables[i]); + }, buildable); + } updateProfile(buildables); From bc90252cec9af05b897cf209012d44a9b20ea251 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 23:08:58 +0100 Subject: [PATCH 310/477] nix profile install: Support installing non-flakes Fixes #4458. --- src/nix/profile.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ca95817d0..765d6866e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -252,8 +252,28 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME manifest.elements.emplace_back(std::move(element)); - } else - throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what()); + } else { + auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + + for (auto & buildable : buildables) { + ProfileElement element; + + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path, {}}); + element.storePaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + pathsToBuild.push_back({bfd.drvPath, {output.first}}); + element.storePaths.insert(output.second); + } + }, + }, buildable); + + manifest.elements.emplace_back(std::move(element)); + } + } } store->buildPaths(pathsToBuild); From 259100332f96250d6615d5839f6a77798c77aefb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 10:29:51 +0100 Subject: [PATCH 311/477] Fix clang build --- src/nix/build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 45f63bb7e..724ce9d79 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (const auto & [i, buildable] : enumerate(buildables)) { + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; From 144cad906991015e997a6b3e7cc69412eb2b8ab1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 18 Jan 2021 18:13:07 +0100 Subject: [PATCH 312/477] narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash This change is to simplify [Trustix](https://github.com/tweag/trustix) indexing and makes it possible to reconstruct this URL regardless of the compression used. In particular this means that https://github.com/tweag/trustix/blob/7c2e9ca597de233846e0b265fb081626ca6c59d8/contrib/nix/nar/nar.go#L61-L71 can be removed and only the bits that are required to establish trust needs to be published in the Trustix build logs. --- src/libstore/binary-cache-store.cc | 6 +----- tests/binary-cache.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4f5f8607d..15163ead5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,11 +176,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); + narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 355a37d97..937585d6f 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -55,7 +55,7 @@ basicTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) +nar=$(ls $cacheDir/nar/*.nar | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar From 8d4268d1901452164b3e666f2eb6bd6bf516493b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:27:36 +0100 Subject: [PATCH 313/477] Improve error formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { --- src/build-remote/build-remote.cc | 52 ++++---- src/libexpr/attr-set.hh | 2 +- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 18 +-- src/libexpr/nixexpr.cc | 2 +- src/libexpr/nixexpr.hh | 2 +- src/libexpr/parser.y | 30 ++--- src/libexpr/primops.cc | 90 ++++++------- src/libexpr/primops/context.cc | 6 +- src/libexpr/primops/fetchMercurial.cc | 4 +- src/libexpr/primops/fetchTree.cc | 6 +- src/libexpr/primops/fromTOML.cc | 2 +- src/libstore/build/derivation-goal.cc | 41 +++--- src/libstore/build/substitution-goal.cc | 7 +- src/libstore/build/worker.cc | 5 +- src/libstore/builtins/buildenv.cc | 10 +- src/libstore/filetransfer.cc | 15 +-- src/libstore/local-store.cc | 32 +---- src/libstore/optimise-store.cc | 17 +-- src/libstore/sqlite.cc | 2 +- src/libutil/error.cc | 169 ++++++++---------------- src/libutil/error.hh | 19 +-- src/libutil/logging.cc | 7 +- src/libutil/serialise.cc | 22 ++- src/libutil/tests/logging.cc | 35 ++--- src/nix-build/nix-build.cc | 7 +- src/nix-env/nix-env.cc | 20 +-- src/nix-store/nix-store.cc | 17 +-- src/nix/daemon.cc | 4 +- src/nix/upgrade-nix.cc | 5 +- src/nix/verify.cc | 19 +-- 31 files changed, 249 insertions(+), 422 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8348d8c91..a4cf91858 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -172,13 +172,14 @@ static int main_build_remote(int argc, char * * argv) else { // build the hint template. - string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)"; - hintstring += "\n%s available machines:"; - hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; + string errorText = + "Failed to find a machine for remote build!\n" + "derivation: %s\nrequired (system, features): (%s, %s)"; + errorText += "\n%s available machines:"; + errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; - for (unsigned int i = 0; i < machines.size(); ++i) { - hintstring += "\n(%s, %s, %s, %s)"; - } + for (unsigned int i = 0; i < machines.size(); ++i) + errorText += "\n(%s, %s, %s, %s)"; // add the template values. string drvstr; @@ -187,25 +188,21 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto hint = hintformat(hintstring); - hint - % drvstr - % neededSystem - % concatStringsSep(", ", requiredFeatures) - % machines.size(); + auto error = hintformat(errorText); + error + % drvstr + % neededSystem + % concatStringsSep(", ", requiredFeatures) + % machines.size(); - for (auto & m : machines) { - hint % concatStringsSep>(", ", m.systemTypes) - % m.maxJobs - % concatStringsSep(", ", m.supportedFeatures) - % concatStringsSep(", ", m.mandatoryFeatures); - } + for (auto & m : machines) + error + % concatStringsSep>(", ", m.systemTypes) + % m.maxJobs + % concatStringsSep(", ", m.supportedFeatures) + % concatStringsSep(", ", m.mandatoryFeatures); - logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, { - .name = "Remote build", - .description = "Failed to find a machine for remote build!", - .hint = hint - }); + printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } @@ -230,12 +227,9 @@ static int main_build_remote(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - logError({ - .name = "Remote build", - .hint = hintfmt("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)) - }); + printError("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7eaa16c59..6d68e5df3 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -77,7 +77,7 @@ public: auto a = get(name); if (!a) throw Error({ - .hint = hintfmt("attribute '%s' missing", name), + .msg = hintfmt("attribute '%s' missing", name), .errPos = pos }); diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f6dead6b0..655408cd3 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -10,7 +10,7 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { throw EvalError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } @@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { throw TypeError({ - .hint = hintfmt(s, showType(v)), + .msg = hintfmt(s, showType(v)), .errPos = pos }); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f3471aac7..7271776eb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -622,7 +622,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { throw EvalError({ - .hint = hintfmt(s, s2), + .msg = hintfmt(s, s2), .errPos = pos }); } @@ -635,7 +635,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { throw EvalError({ - .hint = hintfmt(s, s2, s3), + .msg = hintfmt(s, s2, s3), .errPos = pos }); } @@ -644,7 +644,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const { // p1 is where the error occurred; p2 is a position mentioned in the message. throw EvalError({ - .hint = hintfmt(s, sym, p2), + .msg = hintfmt(s, sym, p2), .errPos = p1 }); } @@ -652,7 +652,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { throw TypeError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } @@ -660,7 +660,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ - .hint = hintfmt(s, fun.showNamePos(), s2), + .msg = hintfmt(s, fun.showNamePos(), s2), .errPos = pos }); } @@ -668,7 +668,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { throw AssertionError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -676,7 +676,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { throw UndefinedVarError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -684,7 +684,7 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) { throw MissingArgumentError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -2057,7 +2057,7 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError({ - .hint = hintfmt("cannot coerce %1% to a string", showType()), + .msg = hintfmt("cannot coerce %1% to a string", showType()), .errPos = pos }); } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d5698011f..492b819e7 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env) "undefined variable" error now. */ if (withLevel == -1) throw UndefinedVarError({ - .hint = hintfmt("undefined variable '%1%'", name), + .msg = hintfmt("undefined variable '%1%'", name), .errPos = pos }); fromWith = true; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 530202ff6..cbe9a45bf 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -239,7 +239,7 @@ struct ExprLambda : Expr { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .msg = hintfmt("duplicate formal function argument '%1%'", arg), .errPos = pos }); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 85eb05d61..49d995bb9 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -32,7 +32,7 @@ namespace nix { Path basePath; Symbol file; FileOrigin origin; - ErrorInfo error; + std::optional error; Symbol sLetBody; ParseData(EvalState & state) : state(state) @@ -66,8 +66,8 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(attrPath), prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), .errPos = pos }); } @@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), .errPos = pos }); } @@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", + .msg = hintfmt("duplicate formal function argument '%1%'", formal.name), .errPos = pos }); @@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { data->error = { - .hint = hintfmt(error), + .msg = hintfmt(error), .errPos = makeCurPos(*loc, data) }; } @@ -338,7 +338,7 @@ expr_function | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in let"), + .msg = hintfmt("dynamic attributes not allowed in let"), .errPos = CUR_POS }); $$ = new ExprLet($2, $4); @@ -418,7 +418,7 @@ expr_simple static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) throw ParseError({ - .hint = hintfmt("URL literals are disabled"), + .msg = hintfmt("URL literals are disabled"), .errPos = CUR_POS }); $$ = new ExprString(data->symbols.create($1)); @@ -491,7 +491,7 @@ attrs delete str; } else throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in inherit"), + .msg = hintfmt("dynamic attributes not allowed in inherit"), .errPos = makeCurPos(@2, data) }); } @@ -576,7 +576,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, ParseData data(*this); data.origin = origin; switch (origin) { - case foFile: + case foFile: data.file = data.symbols.create(path); break; case foStdin: @@ -593,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) throw ParseError(data.error); + if (res) throw ParseError(data.error.value()); data.result->bindVars(staticEnv); @@ -703,7 +703,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos return corepkgsPrefix + path.substr(4); throw ThrownError({ - .hint = hintfmt(evalSettings.pureEval + .msg = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), @@ -725,8 +725,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ - .name = "Entry download", - .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } @@ -736,8 +735,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl res = { true, path }; else { logWarning({ - .name = "Entry not found", - .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c73a94f4e..a470ed6df 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -115,7 +115,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -282,7 +282,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -322,7 +322,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) { throw EvalError({ - .hint = hintfmt("at least one argument to 'exec' required"), + .msg = hintfmt("at least one argument to 'exec' required"), .errPos = pos }); } @@ -336,7 +336,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", program, e.path), .errPos = pos }); @@ -551,7 +551,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'startSet' required"), + .msg = hintfmt("attribute 'startSet' required"), .errPos = pos }); state.forceList(*startSet->value, pos); @@ -565,7 +565,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'operator' required"), + .msg = hintfmt("attribute 'operator' required"), .errPos = pos }); state.forceValue(*op->value, pos); @@ -587,7 +587,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'key' required"), + .msg = hintfmt("attribute 'key' required"), .errPos = pos }); state.forceValue(*key->value, pos); @@ -810,7 +810,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("required attribute 'name' missing"), + .msg = hintfmt("required attribute 'name' missing"), .errPos = pos }); string drvName; @@ -859,7 +859,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else throw EvalError({ - .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .errPos = posDrvName }); }; @@ -869,7 +869,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : ss) { if (outputs.find(j) != outputs.end()) throw EvalError({ - .hint = hintfmt("duplicate derivation output '%1%'", j), + .msg = hintfmt("duplicate derivation output '%1%'", j), .errPos = posDrvName }); /* !!! Check whether j is a valid attribute @@ -879,14 +879,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * the resulting set. */ if (j == "drv") throw EvalError({ - .hint = hintfmt("invalid derivation output name 'drv'" ), + .msg = hintfmt("invalid derivation output name 'drv'" ), .errPos = posDrvName }); outputs.insert(j); } if (outputs.empty()) throw EvalError({ - .hint = hintfmt("derivation cannot have an empty set of outputs"), + .msg = hintfmt("derivation cannot have an empty set of outputs"), .errPos = posDrvName }); }; @@ -1007,20 +1007,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") throw EvalError({ - .hint = hintfmt("required attribute 'builder' missing"), + .msg = hintfmt("required attribute 'builder' missing"), .errPos = posDrvName }); if (drv.platform == "") throw EvalError({ - .hint = hintfmt("required attribute 'system' missing"), + .msg = hintfmt("required attribute 'system' missing"), .errPos = posDrvName }); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) throw EvalError({ - .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .errPos = posDrvName }); @@ -1031,7 +1031,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error({ - .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), .errPos = posDrvName }); @@ -1211,7 +1211,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) throw EvalError({ - .hint = hintfmt("path '%1%' is not in the Nix store", path), + .msg = hintfmt("path '%1%' is not in the Nix store", path), .errPos = pos }); auto path2 = state.store->toStorePath(path).first; @@ -1247,7 +1247,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot check the existence of '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -1324,7 +1324,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1363,7 +1363,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'path' missing"), + .msg = hintfmt("attribute 'path' missing"), .errPos = pos }); @@ -1374,7 +1374,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1400,7 +1400,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va std::optional ht = parseHashType(type); if (!ht) throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash type '%1%'", type), .errPos = pos }); @@ -1430,7 +1430,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val state.realiseContext(ctx); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1650,7 +1650,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') throw EvalError( { - .hint = hintfmt( + .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", name, path), @@ -1801,14 +1801,14 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = pos }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) throw TypeError({ - .hint = hintfmt( + .msg = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", showType(*args[0])), .errPos = pos @@ -1875,7 +1875,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = *attr.pos }); } else if (attr.name == state.sName) @@ -1889,13 +1889,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else throw EvalError({ - .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), .errPos = *attr.pos }); } if (path.empty()) throw EvalError({ - .hint = hintfmt("'path' required"), + .msg = hintfmt("'path' required"), .errPos = pos }); if (name.empty()) @@ -2010,7 +2010,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute '%1%' missing", attr), + .msg = hintfmt("attribute '%1%' missing", attr), .errPos = pos }); // !!! add to stack trace? @@ -2142,7 +2142,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) throw TypeError({ - .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), .errPos = pos }); string name = state.forceStringNoCtx(*j->value, pos); @@ -2152,7 +2152,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) throw TypeError({ - .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), .errPos = pos }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); @@ -2258,7 +2258,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } if (!args[0]->isLambda()) throw TypeError({ - .hint = hintfmt("'functionArgs' requires a function"), + .msg = hintfmt("'functionArgs' requires a function"), .errPos = pos }); @@ -2352,7 +2352,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error({ - .hint = hintfmt("list index %1% is out of bounds", n), + .msg = hintfmt("list index %1% is out of bounds", n), .errPos = pos }); state.forceValue(*list.listElems()[n], pos); @@ -2400,7 +2400,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value state.forceList(*args[0], pos); if (args[0]->listSize() == 0) throw Error({ - .hint = hintfmt("'tail' called on an empty list"), + .msg = hintfmt("'tail' called on an empty list"), .errPos = pos }); @@ -2639,7 +2639,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val if (len < 0) throw EvalError({ - .hint = hintfmt("cannot create list of size %1%", len), + .msg = hintfmt("cannot create list of size %1%", len), .errPos = pos }); @@ -2890,7 +2890,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixFloat f2 = state.forceFloat(*args[1], pos); if (f2 == 0) throw EvalError({ - .hint = hintfmt("division by zero"), + .msg = hintfmt("division by zero"), .errPos = pos }); @@ -2902,7 +2902,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) throw EvalError({ - .hint = hintfmt("overflow in integer division"), + .msg = hintfmt("overflow in integer division"), .errPos = pos }); @@ -3033,7 +3033,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V if (start < 0) throw EvalError({ - .hint = hintfmt("negative start position in 'substring'"), + .msg = hintfmt("negative start position in 'substring'"), .errPos = pos }); @@ -3084,7 +3084,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, std::optional ht = parseHashType(type); if (!ht) throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash type '%1%'", type), .errPos = pos }); @@ -3148,12 +3148,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } @@ -3256,12 +3256,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } @@ -3341,7 +3341,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) throw EvalError({ - .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), .errPos = pos }); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index b570fca31..31cf812b4 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) throw EvalError({ - .hint = hintfmt("Context key '%s' is not a store path", i.name), + .msg = hintfmt("Context key '%s' is not a store path", i.name), .errPos = *i.pos }); if (!settings.readOnlyMode) @@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } @@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 845a1ed1b..4830ebec3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -38,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), .errPos = *attr.pos }); } if (url.empty()) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ab80be2d3..48598acaf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -115,7 +115,7 @@ static void fetchTree( if (!attrs.count("type")) throw Error({ - .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .errPos = pos }); @@ -177,14 +177,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), .errPos = *attr.pos }); } if (!url) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); } else diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 77bff44ae..4c6682dfd 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -82,7 +82,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { throw EvalError({ - .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos }); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..c733ccf08 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -87,8 +87,8 @@ void handleDiffHook( printError(chomp(diffRes.second)); } catch (Error & error) { ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); + // FIXME: wrap errors. + ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } @@ -439,12 +439,9 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); + printError( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); @@ -877,9 +874,12 @@ void DerivationGoal::buildDone() statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; + msg += fmt(";\nlast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } } if (diskFull) @@ -1055,12 +1055,9 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); + printError( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); worker.hook = 0; return rpDecline; } else @@ -3068,10 +3065,7 @@ void DerivationGoal::registerOutputs() auto rewriteOutput = [&]() { /* Apply hash rewriting if necessary. */ if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); + warn("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: this is in-memory. */ StringSink sink; @@ -3359,10 +3353,7 @@ void DerivationGoal::registerOutputs() if (settings.enforceDeterminism) throw NotDeterministic(hint); - logError({ - .name = "Output determinism error", - .hint = hint - }); + printError(hint); curRound = nrRounds; // we know enough, bail out early } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..760fd8ab8 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,11 +146,8 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); + warn("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)); tryNext(); return; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..880a93b15 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -454,10 +454,7 @@ bool Worker::pathContentsGood(const StorePath & path) } pathContentsGoodCache.insert_or_assign(path, res); if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); + printError("path '%s' is corrupted or missing!", store.printStorePath(path)); return res; } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 802fb87bc..e88fc687a 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - directory", - .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) - }); + warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } throw; @@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - skipping symlink", - .hint = hintfmt("skipping dangling symlink '%s'", dstFile) - }); + warn("skipping dangling symlink '%s'", dstFile); continue; } throw; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..677ad44cc 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -632,11 +632,7 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - logError({ - .name = "File transfer", - .hint = hintfmt("unexpected error in download thread: %s", - e.what()) - }); + printError("unexpected error in download thread: %s", e.what()); } { @@ -852,11 +848,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. - if (response && (response->size() < 1024 || response->find("") != string::npos)) { - err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); - } else { - err.hint = hf; - } + if (response && (response->size() < 1024 || response->find("") != string::npos)) + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + else + err.msg = hf; } bool isUri(const string & s) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ab78f1435..f306d8505 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -150,12 +150,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - logError({ - .name = "'build-users-group' not found", - .hint = hintfmt( - "warning: the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup) - }); + printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) @@ -1403,12 +1398,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - logError({ - .name = "Invalid hash", - .hint = hintfmt( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash) - }); + printError("link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash); if (repair) { if (unlink(linkPath.c_str()) == 0) printInfo("removed link '%s'", linkPath); @@ -1441,11 +1432,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - logError({ - .name = "Invalid hash - path modified", - .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -1496,10 +1484,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - logError({ - .name = "Nix path not found", - .hint = hintfmt("path '%s' is not in the Nix store", pathS) - }); + printError("path '%s' is not in the Nix store", pathS); return; } @@ -1522,10 +1507,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - logError({ - .name = "Missing path with referrers", - .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) - }); + printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { repairPath(path); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a0d482ddf..78d587139 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -126,16 +126,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - logWarning({ - .name = "Suspicious file", - .hint = hintfmt("skipping suspicious writable file '%1%'", path) - }); + warn("skipping suspicious writable file '%1%'", path); return; } /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2); return; } @@ -191,10 +188,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - logWarning({ - .name = "Corrupted link", - .hint = hintfmt("removing corrupted link '%1%'", linkPath) - }); + warn("removing corrupted link '%s'", linkPath); unlink(linkPath.c_str()); goto retry; } @@ -229,10 +223,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - logError({ - .name = "Unlink error", - .hint = hintfmt("unable to unlink '%1%'", tempLink) - }); + printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index f5935ee5c..447b4179b 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -211,7 +211,7 @@ void handleSQLiteBusy(const SQLiteBusy & e) lastWarned = now; logWarning({ .name = "Sqlite busy", - .hint = hintfmt(e.what()) + .msg = hintfmt(e.what()) }); } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7dc3f1d3..bc5f9e440 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -204,168 +204,109 @@ void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) } } +static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s) +{ + std::string res; + bool first = true; + + while (!s.empty()) { + auto end = s.find('\n'); + if (!first) res += "\n"; + res += first ? indentFirst : indentRest; + res += s.substr(0, end); + first = false; + if (end == s.npos) break; + s = s.substr(end + 1); + } + + return res; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { - auto errwidth = std::max(getWindowSize().second, 20); - string prefix = ""; - - string levelString; + std::string prefix; switch (einfo.level) { case Verbosity::lvlError: { - levelString = ANSI_RED; - levelString += "error:"; - levelString += ANSI_NORMAL; + prefix = ANSI_RED "error"; + break; + } + case Verbosity::lvlNotice: { + prefix = ANSI_RED "note"; break; } case Verbosity::lvlWarn: { - levelString = ANSI_YELLOW; - levelString += "warning:"; - levelString += ANSI_NORMAL; + prefix = ANSI_YELLOW "warning"; break; } case Verbosity::lvlInfo: { - levelString = ANSI_GREEN; - levelString += "info:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "info"; break; } case Verbosity::lvlTalkative: { - levelString = ANSI_GREEN; - levelString += "talk:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "talk"; break; } case Verbosity::lvlChatty: { - levelString = ANSI_GREEN; - levelString += "chat:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "chat"; break; } case Verbosity::lvlVomit: { - levelString = ANSI_GREEN; - levelString += "vomit:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "vomit"; break; } case Verbosity::lvlDebug: { - levelString = ANSI_YELLOW; - levelString += "debug:"; - levelString += ANSI_NORMAL; - break; - } - default: { - levelString = fmt("invalid error level: %1%", einfo.level); + prefix = ANSI_YELLOW "debug"; break; } + default: + assert(false); } - auto ndl = prefix.length() - + filterANSIEscapes(levelString, true).length() - + 7 - + einfo.name.length() - + einfo.programName.value_or("").length(); - auto dashwidth = std::max(errwidth - ndl, 3); - - std::string dashes(dashwidth, '-'); - - // divider. - if (einfo.name != "") - out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, - prefix, - levelString, - einfo.name, - dashes, - einfo.programName.value_or("")); + // FIXME: show the program name as part of the trace? + if (einfo.programName && einfo.programName != ErrorInfo::programName) + prefix += fmt(" [%s]:" ANSI_NORMAL " ", einfo.programName.value_or("")); else - out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, - prefix, - levelString, - dashes, - einfo.programName.value_or("")); + prefix += ":" ANSI_NORMAL " "; - bool nl = false; // intersperse newline between sections. - if (einfo.errPos.has_value() && (*einfo.errPos)) { - out << prefix << std::endl; - printAtPos(prefix, *einfo.errPos, out); - nl = true; - } + std::ostringstream oss; + oss << einfo.msg << "\n"; - // description - if (einfo.description != "") { - if (nl) - out << std::endl << prefix; - out << std::endl << prefix << einfo.description; - nl = true; - } + if (einfo.errPos.has_value() && *einfo.errPos) { + oss << "\n"; + printAtPos("", *einfo.errPos, oss); - if (einfo.errPos.has_value() && (*einfo.errPos)) { auto loc = getCodeLines(*einfo.errPos); // lines of code. if (loc.has_value()) { - if (nl) - out << std::endl << prefix; - printCodeLines(out, prefix, *einfo.errPos, *loc); - nl = true; + oss << "\n"; + printCodeLines(oss, "", *einfo.errPos, *loc); + oss << "\n"; } } - // hint - if (einfo.hint.has_value()) { - if (nl) - out << std::endl << prefix; - out << std::endl << prefix << *einfo.hint; - nl = true; - } - // traces - if (showTrace && !einfo.traces.empty()) - { - const string tracetitle(" show-trace "); - - int fill = errwidth - tracetitle.length(); - int lw = 0; - int rw = 0; - const int min_dashes = 3; - if (fill > min_dashes * 2) { - if (fill % 2 != 0) { - lw = fill / 2; - rw = lw + 1; - } - else - { - lw = rw = fill / 2; - } - } - else - lw = rw = min_dashes; - - if (nl) - out << std::endl << prefix; - - out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL; - - for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) - { - out << std::endl << prefix; - out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str(); + if (showTrace && !einfo.traces.empty()) { + for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { + oss << "\n" << "… " << iter->hint.str() << "\n"; if (iter->pos.has_value() && (*iter->pos)) { auto pos = iter->pos.value(); - out << std::endl << prefix; - printAtPos(prefix, pos, out); + oss << "\n"; + printAtPos("", pos, oss); auto loc = getCodeLines(pos); - if (loc.has_value()) - { - out << std::endl << prefix; - printCodeLines(out, prefix, pos, *loc); - out << std::endl << prefix; + if (loc.has_value()) { + oss << "\n"; + printCodeLines(oss, "", pos, *loc); + oss << "\n"; } } } } + out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str())); + return out; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e0bde7ea..ff58d3e00 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -107,9 +107,8 @@ struct Trace { struct ErrorInfo { Verbosity level; - string name; - string description; // FIXME: remove? it seems to be barely used - std::optional hint; + string name; // FIXME: rename + hintformat msg; std::optional errPos; std::list traces; @@ -133,23 +132,17 @@ public: template BaseError(unsigned int status, const Args & ... args) - : err {.level = lvlError, - .hint = hintfmt(args...) - } + : err { .level = lvlError, .msg = hintfmt(args...) } , status(status) { } template BaseError(const std::string & fs, const Args & ... args) - : err {.level = lvlError, - .hint = hintfmt(fs, args...) - } + : err { .level = lvlError, .msg = hintfmt(fs, args...) } { } BaseError(hintformat hint) - : err {.level = lvlError, - .hint = hint - } + : err { .level = lvlError, .msg = hint } { } BaseError(ErrorInfo && e) @@ -206,7 +199,7 @@ public: { errNo = errno; auto hf = hintfmt(args...); - err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } virtual const char* sname() const override { return "SysError"; } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6fd0dacef..d2e801175 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -184,7 +184,7 @@ struct JSONLogger : Logger { json["action"] = "msg"; json["level"] = ei.level; json["msg"] = oss.str(); - json["raw_msg"] = ei.hint->str(); + json["raw_msg"] = ei.msg.str(); if (ei.errPos.has_value() && (*ei.errPos)) { json["line"] = ei.errPos->line; @@ -305,10 +305,7 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - logError({ - .name = "JSON log message", - .hint = hintfmt("bad log message from builder: %s", e.what()) - }); + printError("bad JSON log message from builder: %s", e.what()); } return true; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 87c1099a1..d1a16b6ba 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -52,10 +52,7 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - logWarning({ - .name = "Large path", - .description = "dumping very large path (> 256 MiB); this may run out of memory" - }); + warn("dumping very large path (> 256 MiB); this may run out of memory"); } @@ -306,8 +303,7 @@ Sink & operator << (Sink & sink, const Error & ex) << "Error" << info.level << info.name - << info.description - << (info.hint ? info.hint->str() : "") + << info.msg.str() << 0 // FIXME: info.errPos << info.traces.size(); for (auto & trace : info.traces) { @@ -374,12 +370,14 @@ Error readError(Source & source) { auto type = readString(source); assert(type == "Error"); - ErrorInfo info; - info.level = (Verbosity) readInt(source); - info.name = readString(source); - info.description = readString(source); - auto hint = readString(source); - if (hint != "") info.hint = hintformat(std::move(format("%s") % hint)); + auto level = (Verbosity) readInt(source); + auto name = readString(source); + auto msg = readString(source); + ErrorInfo info { + .level = level, + .name = name, + .msg = hintformat(std::move(format("%s") % msg)), + }; auto havePos = readNum(source); assert(havePos == 0); auto nrTraces = readNum(source); diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 5b32c84a4..d990e5499 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -1,3 +1,5 @@ +#if 0 + #include "logging.hh" #include "nixexpr.hh" #include "util.hh" @@ -41,8 +43,7 @@ namespace nix { makeJSONLogger(*logger)->logEI({ .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -62,7 +63,7 @@ namespace nix { throw TestError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : "")); + ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str())); testing::internal::CaptureStderr(); logger->logEI(ei); @@ -95,7 +96,6 @@ namespace nix { logger->logEI({ .level = lvlInfo, .name = "Info name", - .description = "Info description", }); auto str = testing::internal::GetCapturedStderr(); @@ -109,7 +109,6 @@ namespace nix { logger->logEI({ .level = lvlTalkative, .name = "Talkative name", - .description = "Talkative description", }); auto str = testing::internal::GetCapturedStderr(); @@ -123,7 +122,6 @@ namespace nix { logger->logEI({ .level = lvlChatty, .name = "Chatty name", - .description = "Talkative description", }); auto str = testing::internal::GetCapturedStderr(); @@ -137,7 +135,6 @@ namespace nix { logger->logEI({ .level = lvlDebug, .name = "Debug name", - .description = "Debug description", }); auto str = testing::internal::GetCapturedStderr(); @@ -151,7 +148,6 @@ namespace nix { logger->logEI({ .level = lvlVomit, .name = "Vomit name", - .description = "Vomit description", }); auto str = testing::internal::GetCapturedStderr(); @@ -167,7 +163,6 @@ namespace nix { logError({ .name = "name", - .description = "error description", }); auto str = testing::internal::GetCapturedStderr(); @@ -182,8 +177,7 @@ namespace nix { logError({ .name = "error name", - .description = "error with code lines", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foString, problem_file, 02, 13), @@ -200,8 +194,7 @@ namespace nix { logError({ .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -216,7 +209,7 @@ namespace nix { logError({ .name = "error name", - .hint = hintfmt("hint %1%", "only"), + .msg = hintfmt("hint %1%", "only"), }); auto str = testing::internal::GetCapturedStderr(); @@ -233,8 +226,7 @@ namespace nix { logWarning({ .name = "name", - .description = "warning description", - .hint = hintfmt("there was a %1%", "warning"), + .msg = hintfmt("there was a %1%", "warning"), }); auto str = testing::internal::GetCapturedStderr(); @@ -250,8 +242,7 @@ namespace nix { logWarning({ .name = "warning name", - .description = "warning description", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foStdin, problem_file, 2, 13), @@ -274,8 +265,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "show-traces", - .hint = hintfmt("it has been %1% days since our last error", "zero"), + .msg = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -301,8 +291,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "hide traces", - .hint = hintfmt("it has been %1% days since our last error", "zero"), + .msg = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -377,3 +366,5 @@ namespace nix { } } + +#endif diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 38048da52..d1c14596c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -369,11 +369,8 @@ static void main_nix_build(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - logWarning({ - .name = "bashInteractive", - .hint = hintfmt("%s; will use bash from your environment", - (e.info().hint ? e.info().hint->str() : "")) - }); + logError(e.info()); + notice("will use bash from your environment"); shell = "bash"; } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 9963f05d9..d6a16999f 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -124,10 +124,7 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - logError({ - .name = "Name collision", - .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) - }); + printError("warning: name collision in input Nix expressions, skipping '%1%'", path2); continue; } /* Load the expression on demand. */ @@ -876,11 +873,7 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - logError({ - .name = "Invalid meta attribute", - .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) - }); + printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); placeholder.write(nullptr); } else { PathSet context; @@ -1131,12 +1124,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["name"] = j; Value * v = i.queryMeta(j); if (!v) - logError({ - .name = "Invalid meta attribute", - .hint = hintfmt( - "derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) - }); + printError( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); else { if (v->type() == nString) { attrs2["type"] = "string"; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b97f684a4..b7eda5ba6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -708,10 +708,7 @@ static void opVerify(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - logWarning({ - .name = "Store consistency", - .description = "not all errors were fixed" - }); + warn("not all store errors were fixed"); throw Exit(1); } } @@ -733,14 +730,10 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - logError({ - .name = "Hash mismatch", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(Base32, true), + current.first.to_string(Base32, true)); status = 1; } } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 204d4ce6b..a358cb0d9 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -258,8 +258,8 @@ static void daemonLoop() return; } catch (Error & error) { ErrorInfo ei = error.info(); - ei.hint = std::optional(hintfmt("error processing connection: %1%", - (error.info().hint.has_value() ? error.info().hint->str() : ""))); + // FIXME: add to trace? + ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); logError(ei); } } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 299ea40aa..9cd567896 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -61,10 +61,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - logWarning({ - .name = "Version update", - .hint = hintfmt("would upgrade to version %s", version) - }); + warn("would upgrade to version %s", version); return; } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index b2963cf74..9b04e032a 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -101,14 +101,10 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - logError({ - .name = "Hash error - path modified", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(Base32, true), + hash.first.to_string(Base32, true)); } } @@ -156,12 +152,7 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - logError({ - .name = "Untrusted path", - .hint = hintfmt("path '%s' is untrusted", - store->printStorePath(info->path)) - }); - + printError("path '%s' is untrusted", store->printStorePath(info->path)); } } From 40608342cb3772a6d2a6c125cc2237b97c028ab4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:49:29 +0100 Subject: [PATCH 314/477] Remove trailing whitespace --- src/libutil/error.cc | 3 +-- src/libutil/util.cc | 2 +- src/libutil/util.hh | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index bc5f9e440..ddeb5412a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -212,8 +212,7 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR while (!s.empty()) { auto end = s.find('\n'); if (!first) res += "\n"; - res += first ? indentFirst : indentRest; - res += s.substr(0, end); + res += chomp(std::string(first ? indentFirst : indentRest) + std::string(s.substr(0, end))); first = false; if (end == s.npos) break; s = s.substr(end + 1); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e6b6d287d..89f7b58f8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1249,7 +1249,7 @@ template StringSet tokenizeString(std::string_view s, const string & separators) template vector tokenizeString(std::string_view s, const string & separators); -string chomp(const string & s) +string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); return i == string::npos ? "" : string(s, 0, i + 1); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ab0bd865a..ad49c65b3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -373,8 +373,9 @@ template Strings quoteStrings(const C & c) } -/* Remove trailing whitespace from a string. */ -string chomp(const string & s); +/* Remove trailing whitespace from a string. FIXME: return + std::string_view. */ +string chomp(std::string_view s); /* Remove whitespace from the start and end of a string. */ From 55849e153e4b28d03bfca1738c415c438c60f9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:55:59 +0100 Subject: [PATCH 315/477] Change error position formatting It's now at /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix:7:7: instead of at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix The new format is more standard and clickable. --- src/libutil/error.cc | 22 +++++++++------------- tests/misc.sh | 4 ++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index ddeb5412a..5d570a75e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -43,9 +43,9 @@ string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { - return fmt("(%1%:%2%)", errPos.line, errPos.column); + return fmt("%d:%d", errPos.line, errPos.column); } else { - return fmt("(%1%)", errPos.line); + return fmt("%d", errPos.line); } } else { @@ -178,24 +178,20 @@ void printCodeLines(std::ostream & out, } } -void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) +void printAtPos(const ErrPos & pos, std::ostream & out) { - if (pos) - { + if (pos) { switch (pos.origin) { case foFile: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); break; } case foString: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " from string" << ANSI_NORMAL; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } case foStdin: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " from stdin" << ANSI_NORMAL; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } default: @@ -272,7 +268,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (einfo.errPos.has_value() && *einfo.errPos) { oss << "\n"; - printAtPos("", *einfo.errPos, oss); + printAtPos(*einfo.errPos, oss); auto loc = getCodeLines(*einfo.errPos); @@ -292,7 +288,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (iter->pos.has_value() && (*iter->pos)) { auto pos = iter->pos.value(); oss << "\n"; - printAtPos("", pos, oss); + printAtPos(pos, oss); auto loc = getCodeLines(pos); if (loc.has_value()) { diff --git a/tests/misc.sh b/tests/misc.sh index a81c9dbb1..2830856ae 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -17,10 +17,10 @@ nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) -echo $eval_arg_res | grep "at: (1:15) from string" +echo $eval_arg_res | grep "at «string»:1:15:" echo $eval_arg_res | grep "infinite recursion encountered" eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true) -echo $eval_stdin_res | grep "at: (1:15) from stdin" +echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" From 0eb22db3116585821096b7b81295d4bbf5550343 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 12:46:22 +0100 Subject: [PATCH 316/477] Fix macOS build --- .../resolve-system-dependencies.cc | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index d30227e4e..27cf53a45 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,18 +39,12 @@ std::set runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - logError({ - .name = "Regular MACH file", - .hint = hintfmt("file '%s' is not a regular file", filename) - }); + printError("file '%s' is not a regular MACH binary", filename); return {}; } if (st.st_size < sizeof(mach_header_64)) { - logError({ - .name = "File too short", - .hint = hintfmt("file '%s' is too short for a MACH binary", filename) - }); + printError("file '%s' is too short for a MACH binary", filename); return {}; } @@ -72,19 +66,13 @@ std::set runResolver(const Path & filename) } } if (mach64_offset == 0) { - logError({ - .name = "No mach64 blobs", - .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) - }); + printError("could not find any mach64 blobs in file '%1%', continuing...", filename); return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - logError({ - .name = "Magic number", - .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) - }); + printError("Object file has unknown magic number '%1%', skipping it...", magic); return {}; } From d9367a2dd1f2cfe163b9c42e83a0569808ce6fc9 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 21 Jan 2021 17:30:26 +0100 Subject: [PATCH 317/477] scripts/install-nix-from-closure: only show progress if a terminal is used While the progress dots during the copying of the store work fine on a normal terminal, those look pretty off if the script is run inside a provisioning script of e.g. `vagrant` or `packer` where `stderr` and `stdout` are captured: default: . default: .. default: . default: . default: . To work around this, the script checks with `-t 0` if it's running on an actual terminal and doesn't show the progress if that's not the case. --- scripts/install-nix-from-closure.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 6352a8fac..0ee7ce5af 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -166,9 +166,15 @@ fi mkdir -p $dest/store printf "copying Nix to %s..." "${dest}/store" >&2 +# Insert a newline if no progress is shown. +if [ ! -t 0 ]; then + echo "" +fi for i in $(cd "$self/store" >/dev/null && echo ./*); do - printf "." >&2 + if [ -t 0 ]; then + printf "." >&2 + fi i_tmp="$dest/store/$i.$$" if [ -e "$i_tmp" ]; then rm -rf "$i_tmp" From b7bfc7ee52dd425e0156f369eb4c05a62358f912 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 14:54:28 +0100 Subject: [PATCH 318/477] Add FIXME --- src/libutil/error.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7dc3f1d3..2a67a730a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -61,6 +61,8 @@ std::optional getCodeLines(const ErrPos & errPos) if (errPos.origin == foFile) { LinesOfCode loc; try { + // FIXME: when running as the daemon, make sure we don't + // open a file to which the client doesn't have access. AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) return {}; From 8c07ed1ddad6595cd679181b0b8d78e09fc6d152 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 15:27:55 +0000 Subject: [PATCH 319/477] Improve documentation and test and requested --- src/libstore/store-api.hh | 6 +++--- tests/binary-cache-build-remote.sh | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3221cf249..9e98eb8f9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -372,9 +372,9 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently well-formed - (e.g. hash content-address or signature) in order to be included in the - given store. + /* Check whether the given valid path info is sufficiently attested, by + either being signed by a trusted public key or content-addressed, in + order to be included in the given store. These same checks would be performed in addToStore, but this allows an earlier failure in the case where dependencies need to be added too, but diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh index ed51164a4..81cd21a4a 100644 --- a/tests/binary-cache-build-remote.sh +++ b/tests/binary-cache-build-remote.sh @@ -7,7 +7,10 @@ clearCacheCache (! nix-build --store "file://$cacheDir" dependencies.nix) # Succeeds with default store as build remote. -nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix +outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) + +# Test that the path exactly exists in the destination store. +nix path-info --store "file://$cacheDir" $outPath # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix From 53a709535b42197a9abd3fe46406bb186ad6c751 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 10:21:12 -0500 Subject: [PATCH 320/477] Apply suggestions from code review Thanks! Co-authored-by: Eelco Dolstra --- src/build-remote/build-remote.cc | 6 +++--- src/libstore/build/derivation-goal.cc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 350bd6cef..68af3e966 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -75,11 +75,11 @@ static int main_build_remote(int argc, char * * argv) /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = "/current-load"; + auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoad; + currentLoad = std::string { localStore->stateDir } + currentLoadName; else - currentLoad = settings.nixStateDir + currentLoad; + currentLoad = settings.nixStateDir + currentLoadName; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 953e241d8..fa8b99118 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3291,7 +3291,7 @@ void DerivationGoal::registerOutputs() auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; if (buildMode == bmCheck) { @@ -3426,7 +3426,7 @@ void DerivationGoal::registerOutputs() { auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; ValidPathInfos infos2; From a76682466062ef2c972d19f259feeef1c46a44a3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 14:46:40 -0600 Subject: [PATCH 321/477] Handle missing etag in 304 Not Modified response GitHub now omits the etag, but 304 implies it matches the one we provided. Just use that one to avoid having an etag-less resource. Fixes #4469 --- src/libstore/filetransfer.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..1b7eae3ec 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -375,6 +375,13 @@ struct curlFileTransfer : public FileTransfer else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; + + // In 2021, GitHub responds to If-None-Match with 304, + // but omits ETag. We just use the If-None-Match etag + // since 304 implies they are the same. + if (httpStatus == 304 && result.etag == "") + result.etag = request.expectedETag; + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); From 1ea5f0b66ca43eb1f6c552b59de170d61bcf540c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 23:19:52 -0600 Subject: [PATCH 322/477] Remove expectedETag assert in tarball.cc --- src/libfetchers/tarball.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 56c014a8c..b8d7d2c70 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -64,7 +64,6 @@ DownloadFileResult downloadFile( if (res.cached) { assert(cached); - assert(request.expectedETag == res.etag); storePath = std::move(cached->storePath); } else { StringSink sink; From b159d23800eec55412621a0b3e6c926a1dbb1755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 14:38:15 +0100 Subject: [PATCH 323/477] Make '--help' do the same as 'help' (i.e. show a manpage) --- src/libutil/args.cc | 89 --------------------------------------------- src/libutil/args.hh | 14 ------- src/nix/command.cc | 5 --- src/nix/command.hh | 2 - src/nix/main.cc | 61 +++++++++---------------------- src/nix/nar.cc | 5 --- src/nix/store.cc | 5 --- 7 files changed, 17 insertions(+), 164 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index fb5cb80fb..2f2e4bb96 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -96,41 +96,6 @@ void Args::parseCmdline(const Strings & _cmdline) processArgs(pendingArgs, true); } -void Args::printHelp(const string & programName, std::ostream & out) -{ - std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); - for (auto & exp : expectedArgs) { - std::cout << renderLabels({exp.label}); - // FIXME: handle arity > 1 - if (exp.handler.arity == ArityAny) std::cout << "..."; - if (exp.optional) std::cout << "?"; - } - std::cout << "\n"; - - auto s = description(); - if (s != "") - std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; - - if (longFlags.size()) { - std::cout << "\n"; - std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; - printFlags(out); - } -} - -void Args::printFlags(std::ostream & out) -{ - Table2 table; - for (auto & flag : longFlags) { - if (hiddenCategories.count(flag.second->category)) continue; - table.push_back(std::make_pair( - (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") - + "--" + flag.first + renderLabels(flag.second->labels), - flag.second->description)); - } - printTable(out, table); -} - bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); @@ -331,28 +296,6 @@ Strings argvToStrings(int argc, char * * argv) return args; } -std::string renderLabels(const Strings & labels) -{ - std::string res; - for (auto label : labels) { - for (auto & c : label) c = std::toupper(c); - res += " " ANSI_ITALIC + label + ANSI_NORMAL; - } - return res; -} - -void printTable(std::ostream & out, const Table2 & table) -{ - size_t max = 0; - for (auto & row : table) - max = std::max(max, filterANSIEscapes(row.first, true).size()); - for (auto & row : table) { - out << " " << row.first - << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') - << row.second << "\n"; - } -} - MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { @@ -376,38 +319,6 @@ MultiCommand::MultiCommand(const Commands & commands) categories[Command::catDefault] = "Available commands"; } -void MultiCommand::printHelp(const string & programName, std::ostream & out) -{ - if (command) { - command->second->printHelp(programName + " " + command->first, out); - return; - } - - out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); - - out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; - printFlags(out); - - std::map>> commandsByCategory; - - for (auto & [name, commandFun] : commands) { - auto command = commandFun(); - commandsByCategory[command->category()].insert_or_assign(name, command); - } - - for (auto & [category, commands] : commandsByCategory) { - out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); - - Table2 table; - for (auto & [name, command] : commands) { - auto descr = command->description(); - if (!descr.empty()) - table.push_back(std::make_pair(name, descr)); - } - printTable(out, table); - } -} - bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { if (Args::processFlag(pos, end)) return true; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3783bc84f..fda7852cd 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -20,8 +20,6 @@ public: wrong. */ void parseCmdline(const Strings & cmdline); - virtual void printHelp(const string & programName, std::ostream & out); - /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } @@ -115,8 +113,6 @@ protected: virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - virtual void printFlags(std::ostream & out); - /* Positional arguments. */ struct ExpectedArg { @@ -223,8 +219,6 @@ public: MultiCommand(const Commands & commands); - void printHelp(const string & programName, std::ostream & out) override; - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; @@ -234,14 +228,6 @@ public: Strings argvToStrings(int argc, char * * argv); -/* Helper function for rendering argument labels. */ -std::string renderLabels(const Strings & labels); - -/* Helper function for printing 2-column tables. */ -typedef std::vector> Table2; - -void printTable(std::ostream & out, const Table2 & table); - struct Completion { std::string completion; std::string description; diff --git a/src/nix/command.cc b/src/nix/command.cc index ba58c7d6b..20eeefe91 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -27,11 +27,6 @@ nix::Commands RegisterCommand::getCommandsFor(const std::vector & p return res; } -void NixMultiCommand::printHelp(const string & programName, std::ostream & out) -{ - MultiCommand::printHelp(programName, out); -} - nlohmann::json NixMultiCommand::toJSON() { // FIXME: use Command::toJSON() as well. diff --git a/src/nix/command.hh b/src/nix/command.hh index f325cd906..791dd0f1e 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -25,8 +25,6 @@ static constexpr Command::Category catNixInstallation = 102; struct NixMultiCommand : virtual MultiCommand, virtual Command { - void printHelp(const string & programName, std::ostream & out) override; - nlohmann::json toJSON() override; }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 80422bd24..77a13c913 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -54,6 +54,8 @@ static bool haveInternet() std::string programPath; char * * savedArgv; +struct HelpRequested { }; + struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; @@ -71,22 +73,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", .description = "Show usage information.", - .handler = {[&]() { if (!completions) showHelpAndExit(); }}, - }); - - addFlag({ - .longName = "help-config", - .description = "Show configuration settings.", - .handler = {[&]() { - std::cout << "The following configuration settings are available:\n\n"; - Table2 tbl; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - tbl.emplace_back(s.first, s.second.description); - printTable(std::cout, tbl); - throw Exit(); - }}, + .handler = {[&]() { throw HelpRequested(); }}, }); addFlag({ @@ -154,33 +141,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs return pos; } - void printFlags(std::ostream & out) override - { - Args::printFlags(out); - std::cout << - "\n" - "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" - "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" - "--help-config' for a list of configuration settings.\n"; - } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - -#if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; -#endif - - std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; - } - - void showHelpAndExit() - { - printHelp(programName, std::cout); - throw Exit(); - } - std::string description() override { return "a tool for reproducible and declarative configuration management"; @@ -298,6 +258,18 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); + } catch (HelpRequested &) { + std::vector subcommand; + MultiCommand * command = &args; + while (command) { + if (command && command->command) { + subcommand.push_back(command->command->first); + command = dynamic_cast(&*command->command->second); + } else + break; + } + showHelp(subcommand); + return; } catch (UsageError &) { if (!completions) throw; } @@ -306,7 +278,8 @@ void mainWrapped(int argc, char * * argv) initPlugins(); - if (!args.command) args.showHelpAndExit(); + if (!args.command) + throw UsageError("no subcommand specified"); if (args.command->first != "repl" && args.command->first != "doctor" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index 0775d3c25..dbb043d9b 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -28,11 +28,6 @@ struct CmdNar : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/store.cc b/src/nix/store.cc index e91bcc503..44e53c7c7 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -21,11 +21,6 @@ struct CmdStore : virtual NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdStore = registerCommand("store"); From a32073e7e839ea92ada602c0a170855a08afc73a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 14:43:16 +0100 Subject: [PATCH 324/477] Add FIXME --- src/libexpr/primops/fetchTree.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 48598acaf..27d8ddf35 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -153,6 +153,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V fetchTree(state, pos, args, v, std::nullopt); } +// FIXME: document static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, From 3ba98ba8f08523e60310cf75ec809bd21d0ce977 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 17:15:38 +0100 Subject: [PATCH 325/477] Tell user to run 'nix log' to get full build logs --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 36bbe46d4..179a010d4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -896,6 +896,8 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } + msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + worker.store.printStorePath(drvPath)); } if (diskFull) From 807d963ee8d23e88f09e28365b045d322530c5aa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 18:19:32 +0100 Subject: [PATCH 326/477] Group subcommands by category --- doc/manual/generate-manpage.nix | 23 +++++++++++++++++++---- doc/manual/utils.nix | 10 +++++++++- src/libutil/args.cc | 5 ++++- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index c2c748464..30152088d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -13,12 +13,27 @@ let + showSynopsis { inherit command; args = def.args; } + (if def.commands or {} != {} then + let + categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands))); + listCommands = cmds: + concatStrings (map (name: + "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n") + (attrNames cmds)); + in "where *subcommand* is one of the following:\n\n" # FIXME: group by category - + concatStrings (map (name: - "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n") - (attrNames def.commands)) - + "\n" + + (if length categories > 1 + then + concatStrings (map + (cat: + "**${toString cat.description}:**\n\n" + + listCommands (filterAttrs (n: v: v.category == cat) def.commands) + + "\n" + ) categories) + + "\n" + else + listCommands def.commands + + "\n") else "") + (if def ? doc then def.doc + "\n\n" diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 50150bf3e..d4b18472f 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,7 +1,15 @@ with builtins; -{ +rec { splitLines = s: filter (x: !isList x) (split "\n" s); concatStrings = concatStringsSep ""; + + # FIXME: O(n^2) + unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; + + nameValuePair = name: value: { inherit name value; }; + + filterAttrs = pred: set: + listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 2f2e4bb96..6d57e1a34 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -341,7 +341,10 @@ nlohmann::json MultiCommand::toJSON() for (auto & [name, commandFun] : commands) { auto command = commandFun(); auto j = command->toJSON(); - j["category"] = categories[command->category()]; + auto cat = nlohmann::json::object(); + cat["id"] = command->category(); + cat["description"] = categories[command->category()]; + j["category"] = std::move(cat); cmds[name] = std::move(j); } From 36c4d6f59247826dde32ad2e6b5a9471a9a1c911 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 19:03:13 +0100 Subject: [PATCH 327/477] Group common options --- doc/manual/generate-manpage.nix | 40 ++++++++++++++++++++------------- src/libexpr/common-eval-args.cc | 7 ++++++ src/libmain/common-args.cc | 6 ++++- src/libmain/common-args.hh | 17 ++++++++++++-- src/libutil/args.cc | 3 +-- src/libutil/args.hh | 2 +- src/nix/command.cc | 9 +++++++- src/nix/command.hh | 2 ++ src/nix/installables.cc | 13 +++++++++++ src/nix/main.cc | 1 + src/nix/sigs.cc | 6 ++--- 11 files changed, 80 insertions(+), 26 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 30152088d..a563c31f8 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,31 +38,39 @@ let + (if def ? doc then def.doc + "\n\n" else "") - + (let s = showFlags def.flags; in + + (let s = showOptions def.flags; in if s != "" - then "# Flags\n\n${s}" + then "# Options\n\n${s}" else "") ; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; - showFlags = flags: - concatStrings - (map (longName: - let flag = flags.${longName}; in - if flag.category or "" != "config" - then - " - `--${longName}`" - + (if flag ? shortName then " / `-${flag.shortName}`" else "") - + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") - + " \n" - + " " + flag.description + "\n\n" - else "") - (attrNames flags)); + showOptions = flags: + let + categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags))); + in + concatStrings (map + (cat: + (if cat != "" + then "**${cat}:**\n\n" + else "") + + concatStrings + (map (longName: + let + flag = flags.${longName}; + in + " - `--${longName}`" + + (if flag ? shortName then " / `-${flag.shortName}`" else "") + + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") + + " \n" + + " " + flag.description + "\n\n" + ) (attrNames (filterAttrs (n: v: v.category == cat) flags)))) + categories); showSynopsis = { command, args }: - "`${command}` [*flags*...] ${concatStringsSep " " + "`${command}` [*option*...] ${concatStringsSep " " (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; processCommand = { command, def, filename }: diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index ffe782454..aa14bf79b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -12,9 +12,12 @@ namespace nix { MixEvalArgs::MixEvalArgs() { + auto category = "Common evaluation options"; + addFlag({ .longName = "arg", .description = "Pass the value *expr* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); @@ -22,6 +25,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "argstr", .description = "Pass the string *string* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -30,6 +34,7 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); @@ -37,6 +42,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "impure", .description = "Allow access to mutable paths and repositories.", + .category = category, .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -45,6 +51,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", + .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index bd5573e5d..ff96ee7d5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -11,18 +11,21 @@ MixCommonArgs::MixCommonArgs(const string & programName) .longName = "verbose", .shortName = 'v', .description = "Increase the logging verbosity level.", + .category = loggingCategory, .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", .description = "Decrease the logging verbosity level.", + .category = loggingCategory, .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", .description = "Set the logging verbosity level to 'debug'.", + .category = loggingCategory, .handler = {[]() { verbosity = lvlDebug; }}, }); @@ -52,6 +55,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", + .category = loggingCategory, .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -66,7 +70,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) }} }); - std::string cat = "config"; + std::string cat = "Options to override configuration settings"; globalConfig.convertToArgs(*this, cat); // Backward compatibility hack: nix-env already had a --system flag. diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 47f341619..8e53a7361 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -4,6 +4,9 @@ namespace nix { +//static constexpr auto commonArgsCategory = "Miscellaneous common options"; +static constexpr auto loggingCategory = "Logging-related options"; + struct MixCommonArgs : virtual Args { string programName; @@ -16,7 +19,12 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); + addFlag({ + .longName = "dry-run", + .description = "Show what this command would do without doing it.", + //.category = commonArgsCategory, + .handler = {&dryRun, true}, + }); } }; @@ -26,7 +34,12 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); + addFlag({ + .longName = "json", + .description = "Produce output in JSON format, suitable for consumption by another program.", + //.category = commonArgsCategory, + .handler = {&json, true}, + }); } }; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6d57e1a34..71bae0504 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -195,8 +195,7 @@ nlohmann::json Args::toJSON() j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") j["description"] = flag->description; - if (flag->category != "") - j["category"] = flag->category; + j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; if (!flag->labels.empty()) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index fda7852cd..b1020b101 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -91,7 +91,7 @@ protected: { } }; - /* Flags. */ + /* Options. */ struct Flag { typedef std::shared_ptr ptr; diff --git a/src/nix/command.cc b/src/nix/command.cc index 20eeefe91..614dee788 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -61,6 +61,7 @@ StorePathsCommand::StorePathsCommand(bool recursive) addFlag({ .longName = "no-recursive", .description = "Apply operation to specified paths only.", + .category = installablesCategory, .handler = {&this->recursive, false}, }); else @@ -68,10 +69,16 @@ StorePathsCommand::StorePathsCommand(bool recursive) .longName = "recursive", .shortName = 'r', .description = "Apply operation to closure of the specified paths.", + .category = installablesCategory, .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "Apply the operation to every store path.", &all); + addFlag({ + .longName = "all", + .description = "Apply the operation to every store path.", + .category = installablesCategory, + .handler = {&all, true}, + }); } void StorePathsCommand::run(ref store) diff --git a/src/nix/command.hh b/src/nix/command.hh index 791dd0f1e..ed6980075 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -23,6 +23,8 @@ static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; +static constexpr auto installablesCategory = "Options that change the interpretation of installables"; + struct NixMultiCommand : virtual MultiCommand, virtual Command { nlohmann::json toJSON() override; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 34ee238bf..4e6bf4a9a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -58,39 +58,47 @@ void completeFlakeInputPath( MixFlakeOptions::MixFlakeOptions() { + auto category = "Common flake-related options"; + addFlag({ .longName = "recreate-lock-file", .description = "Recreate the flake's lock file from scratch.", + .category = category, .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", + .category = category, .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", .description = "Do not write the flake's newly generated lock file.", + .category = category, .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", .description = "Don't allow lookups in the flake registries.", + .category = category, .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", .description = "Commit changes to the flake's lock file.", + .category = category, .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -104,6 +112,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -115,6 +124,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", + .category = category, .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -144,6 +154,7 @@ SourceExprCommand::SourceExprCommand() .longName = "file", .shortName = 'f', .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", + .category = installablesCategory, .labels = {"file"}, .handler = {&file}, .completer = completePath @@ -152,6 +163,7 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "expr", .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", + .category = installablesCategory, .labels = {"expr"}, .handler = {&expr} }); @@ -159,6 +171,7 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "derivation", .description = "Operate on the store derivation rather than its outputs.", + .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 77a13c913..58b643cc5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -80,6 +80,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "print-build-logs", .shortName = 'L', .description = "Print full build logs on standard error.", + .category = loggingCategory, .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 3445182f2..c64b472b6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "Use signatures from specified store.", + .description = "Copy signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -24,7 +24,7 @@ struct CmdCopySigs : StorePathsCommand std::string description() override { - return "copy path signatures from substituters (like binary caches)"; + return "copy store path signatures from substituters"; } void run(ref store, StorePaths storePaths) override @@ -110,7 +110,7 @@ struct CmdSign : StorePathsCommand std::string description() override { - return "sign the specified paths"; + return "sign store paths"; } void run(ref store, StorePaths storePaths) override From f15f0b8e83051cd95dacb2784b004c8272957f30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Jan 2021 10:34:59 +0100 Subject: [PATCH 328/477] Update to lowdown 0.7.9 --- flake.lock | 17 ----------------- flake.nix | 14 ++++++-------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/flake.lock b/flake.lock index 9f8c788ac..6fe52fbfd 100644 --- a/flake.lock +++ b/flake.lock @@ -1,21 +1,5 @@ { "nodes": { - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1598695561, - "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1602702596, @@ -33,7 +17,6 @@ }, "root": { "inputs": { - "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 9addccd63..fedd0e381 100644 --- a/flake.nix +++ b/flake.nix @@ -2,9 +2,9 @@ description = "The purely functional package manager"; inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; + #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; - outputs = { self, nixpkgs, lowdown-src }: + outputs = { self, nixpkgs }: let @@ -200,16 +200,14 @@ }; lowdown = with final; stdenv.mkDerivation { - name = "lowdown-0.7.1"; + name = "lowdown-0.7.9"; - /* src = fetchurl { - url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.1.tar.gz; - hash = "sha512-1daoAQfYD0LdhK6aFhrSQvadjc5GsSPBZw0fJDb+BEHYMBLjqiUl2A7H8N+l0W4YfGKqbsPYSrCy4vct+7U6FQ=="; + url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz; + hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ=="; }; - */ - src = lowdown-src; + #src = lowdown-src; outputs = [ "out" "bin" "dev" ]; From 6af6e41df06f0a8a3b919b4052b41d09f0a97678 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 26 Jan 2021 06:22:24 -0500 Subject: [PATCH 329/477] Move command plugin interface to libnixcmd --- Makefile | 1 + src/build-remote/build-remote.cc | 2 +- src/{nix => libcmd}/command.cc | 0 src/{nix => libcmd}/command.hh | 0 src/{nix => libcmd}/installables.cc | 0 src/{nix => libcmd}/installables.hh | 0 src/{nix => libcmd}/legacy.cc | 0 src/{nix => libcmd}/legacy.hh | 0 src/libcmd/local.mk | 15 +++++++++++++++ src/{nix => libcmd}/markdown.cc | 0 src/{nix => libcmd}/markdown.hh | 0 src/libcmd/nix-cmd.pc.in | 9 +++++++++ src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-copy-closure/nix-copy-closure.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/daemon.cc | 2 +- src/nix/local.mk | 4 ++-- 21 files changed, 36 insertions(+), 11 deletions(-) rename src/{nix => libcmd}/command.cc (100%) rename src/{nix => libcmd}/command.hh (100%) rename src/{nix => libcmd}/installables.cc (100%) rename src/{nix => libcmd}/installables.hh (100%) rename src/{nix => libcmd}/legacy.cc (100%) rename src/{nix => libcmd}/legacy.hh (100%) create mode 100644 src/libcmd/local.mk rename src/{nix => libcmd}/markdown.cc (100%) rename src/{nix => libcmd}/markdown.hh (100%) create mode 100644 src/libcmd/nix-cmd.pc.in diff --git a/Makefile b/Makefile index f80b8bb82..68ec3ab0c 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ makefiles = \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ + src/libcmd/local.mk \ src/nix/local.mk \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 17a0a8373..5b8ab3387 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,7 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; using std::cin; diff --git a/src/nix/command.cc b/src/libcmd/command.cc similarity index 100% rename from src/nix/command.cc rename to src/libcmd/command.cc diff --git a/src/nix/command.hh b/src/libcmd/command.hh similarity index 100% rename from src/nix/command.hh rename to src/libcmd/command.hh diff --git a/src/nix/installables.cc b/src/libcmd/installables.cc similarity index 100% rename from src/nix/installables.cc rename to src/libcmd/installables.cc diff --git a/src/nix/installables.hh b/src/libcmd/installables.hh similarity index 100% rename from src/nix/installables.hh rename to src/libcmd/installables.hh diff --git a/src/nix/legacy.cc b/src/libcmd/legacy.cc similarity index 100% rename from src/nix/legacy.cc rename to src/libcmd/legacy.cc diff --git a/src/nix/legacy.hh b/src/libcmd/legacy.hh similarity index 100% rename from src/nix/legacy.hh rename to src/libcmd/legacy.hh diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk new file mode 100644 index 000000000..ab0e0e43d --- /dev/null +++ b/src/libcmd/local.mk @@ -0,0 +1,15 @@ +libraries += libcmd + +libcmd_NAME = libnixcmd + +libcmd_DIR := $(d) + +libcmd_SOURCES := $(wildcard $(d)/*.cc) + +libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers + +libcmd_LDFLAGS = -llowdown + +libcmd_LIBS = libstore libutil libexpr libmain libfetchers + +$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644)) diff --git a/src/nix/markdown.cc b/src/libcmd/markdown.cc similarity index 100% rename from src/nix/markdown.cc rename to src/libcmd/markdown.cc diff --git a/src/nix/markdown.hh b/src/libcmd/markdown.hh similarity index 100% rename from src/nix/markdown.hh rename to src/libcmd/markdown.hh diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in new file mode 100644 index 000000000..1761a9f41 --- /dev/null +++ b/src/libcmd/nix-cmd.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixcmd +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d1c14596c..361f9730d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -17,7 +17,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; using namespace std::string_literals; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 309970df6..57189d557 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include "fetchers.hh" #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 57092b887..c1769790a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,7 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 10990f7b5..ad2e06067 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "store-api.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d6a16999f..106a78fc4 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -14,7 +14,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3956fef6d..ea2e85eb0 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -10,7 +10,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b7eda5ba6..37191b9e6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "graphml.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a358cb0d9..26006167d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -8,7 +8,7 @@ #include "globals.hh" #include "derivations.hh" #include "finally.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include "daemon.hh" #include diff --git a/src/nix/local.mk b/src/nix/local.mk index 23c08fc86..83b6dd08b 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -14,9 +14,9 @@ nix_SOURCES := \ $(wildcard src/nix-instantiate/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -nix_LIBS = libexpr libmain libfetchers libstore libutil +nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown From d3c428413367a87ab2d27abe9c7f3c379eb12e1c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 5 Jan 2021 10:01:22 +0100 Subject: [PATCH 330/477] Make the error message for missing outputs more useful Don't only show the name of the output, but also the derivation to which this output belongs (as otherwise it's very hard to track back what went wrong) --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01e2fcc7b..9da415c42 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -394,7 +394,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' has no store path mapped to it", outName); + throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); result.insert_or_assign(outName, *optOutPath); } return result; From 9da11bac5797c34b7bb2ee99275befe9c9fb6dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 7 Jan 2021 11:21:43 +0100 Subject: [PATCH 331/477] Fix the error message when a dep is missing Fix a mismatch in the errors thrown when a needed output was missing from an input derivation that was leading to a wrong and quite misleading error message --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..656f92cee 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -539,12 +539,12 @@ void DerivationGoal::inputsRealised() if (!optRealizedInput) throw Error( "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); worker.store.computeFSClosure(*optRealizedInput, inputPaths); } else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); } } } From 8e758d402ba1045c7b8273f8cb1d6d8d917ca52b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 12:06:03 +0100 Subject: [PATCH 332/477] Remove mkFlag() --- src/libmain/shared.cc | 14 ++++++++++---- src/libutil/args.hh | 21 --------------------- src/nix/eval.cc | 6 +++++- src/nix/hash.cc | 43 +++++++++++++++++++++++++++++++++---------- src/nix/ls.cc | 23 ++++++++++++++++++++--- src/nix/path-info.cc | 30 ++++++++++++++++++++++++++---- src/nix/verify.cc | 13 +++++++++++-- 7 files changed, 105 insertions(+), 45 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7e27e95c2..5baaff3e9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -229,11 +229,17 @@ LegacyArgs::LegacyArgs(const std::string & programName, intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "Do not write to the Nix store.", - &settings.readOnlyMode); + addFlag({ + .longName = "readonly-mode", + .description = "Do not write to the Nix store.", + .handler = {&settings.readOnlyMode, true}, + }); - mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", - &gcWarning, false); + addFlag({ + .longName = "no-gc-warning", + .description = "Disable warnings about not using `--add-root`.", + .handler = {&gcWarning, true}, + }); addFlag({ .longName = "store", diff --git a/src/libutil/args.hh b/src/libutil/args.hh index b1020b101..42d8515ef 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -135,27 +135,6 @@ public: void addFlag(Flag && flag); - /* Helper functions for constructing flags / positional - arguments. */ - - void mkFlag(char shortName, const std::string & name, - const std::string & description, bool * dest) - { - mkFlag(shortName, name, description, dest, true); - } - - template - void mkFlag(char shortName, const std::string & longName, const std::string & description, - T * dest, const T & value) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .handler = {[=]() { *dest = value; }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b5049ac65..65d61e005 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,7 +18,11 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); + addFlag({ + .longName = "raw", + .description = "Print strings without quotes or escaping.", + .handler = {&raw, true}, + }); addFlag({ .longName = "apply", diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 79d506ace..4535e4ab0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,18 +19,41 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); - mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); - mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); - mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); + addFlag({ + .longName = "sri", + .description = "Print the hash in SRI format.", + .handler = {&base, SRI}, + }); + + addFlag({ + .longName = "base64", + .description = "Print the hash in base-64 format.", + .handler = {&base, Base64}, + }); + + addFlag({ + .longName = "base32", + .description = "Print the hash in base-32 (Nix-specific) format.", + .handler = {&base, Base32}, + }); + + addFlag({ + .longName = "base16", + .description = "Print the hash in base-16 format.", + .handler = {&base, Base16}, + }); + addFlag(Flag::mkHashTypeFlag("type", &ht)); + #if 0 - mkFlag() - .longName("modulo") - .description("Compute the hash modulo specified the string.") - .labels({"modulus"}) - .dest(&modulus); - #endif + addFlag({ + .longName = "modulo", + .description = "Compute the hash modulo the specified string.", + .labels = {"modulus"}, + .handler = {&modulus}, + }); + #endif\ + expectArgs({ .label = "paths", .handler = {&paths}, diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c0b1ecb32..c1dc9a95b 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,26 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); - mkFlag('l', "long", "Show detailed file information.", &verbose); - mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); + addFlag({ + .longName = "recursive", + .shortName = 'R', + .description = "List subdirectories recursively.", + .handler = {&recursive, true}, + }); + + addFlag({ + .longName = "long", + .shortName = 'l', + .description = "Show detailed file information.", + .handler = {&verbose, true}, + }); + + addFlag({ + .longName = "directory", + .shortName = 'd', + .description = "Show directories rather than their contents.", + .handler = {&showDirectory, true}, + }); } void listText(ref accessor) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0fa88f1bf..518cd5568 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); - mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); - mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); - mkFlag(0, "sigs", "Show signatures.", &showSigs); + addFlag({ + .longName = "size", + .shortName = 's', + .description = "Print the size of the NAR serialisation of each path.", + .handler = {&showSize, true}, + }); + + addFlag({ + .longName = "closure-size", + .shortName = 'S', + .description = "Print the sum of the sizes of the NAR serialisations of the closure of each path.", + .handler = {&showClosureSize, true}, + }); + + addFlag({ + .longName = "human-readable", + .shortName = 'h', + .description = "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", + .handler = {&humanReadable, true}, + }); + + addFlag({ + .longName = "sigs", + .description = "Show signatures.", + .handler = {&showSigs, true}, + }); } std::string description() override diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b04e032a..1721c7f16 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,8 +18,17 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); - mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); + addFlag({ + .longName = "no-contents", + .description = "Do not verify the contents of each store path.", + .handler = {&noContents, true}, + }); + + addFlag({ + .longName = "no-trust", + .description = "Do not verify whether each store path is trusted.", + .handler = {&noTrust, true}, + }); addFlag({ .longName = "substituter", From c03f41055de6f885ade7fa7927bf83fb697a3dba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:02:54 +0100 Subject: [PATCH 333/477] Add traces to errors while updating flake lock file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example: $ nix build --show-trace error: unable to download 'https://api.github.com/repos/NixOS/nixpkgs/commits/no-such-branch': HTTP error 422 ('') response body: { "message": "No commit found for SHA: no-such-branch", "documentation_url": "https://docs.github.com/rest/reference/repos#get-a-commit" } … while fetching the input 'github:NixOS/nixpkgs/no-such-branch' … while updating the flake input 'nixpkgs' … while updating the lock file of flake 'git+file:///home/eelco/Dev/nix' --- src/libexpr/flake/flake.cc | 474 +++++++++++++++++++----------------- src/libfetchers/fetchers.cc | 9 +- 2 files changed, 252 insertions(+), 231 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0786fef3d..2e94490d4 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -298,284 +298,298 @@ LockedFlake lockFlake( auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); - // FIXME: symlink attack - auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + try { - debug("old lock file: %s", oldLockFile); + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); - // FIXME: check whether all overrides are used. - std::map overrides; - std::set overridesUsed, updatesUsed; + debug("old lock file: %s", oldLockFile); - for (auto & i : lockFlags.inputOverrides) - overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + // FIXME: check whether all overrides are used. + std::map overrides; + std::set overridesUsed, updatesUsed; - LockFile newLockFile; + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); - std::vector parents; + LockFile newLockFile; - std::function node, - const InputPath & inputPathPrefix, - std::shared_ptr oldNode)> - computeLocks; + std::vector parents; - computeLocks = [&]( - const FlakeInputs & flakeInputs, - std::shared_ptr node, - const InputPath & inputPathPrefix, - std::shared_ptr oldNode) - { - debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + std::function node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode)> + computeLocks; - /* Get the overrides (i.e. attributes of the form - 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - // FIXME: check this - for (auto & [id, input] : flake.inputs) { - for (auto & [idOverride, inputOverride] : input.overrides) { + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { auto inputPath(inputPathPrefix); inputPath.push_back(id); - inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); - } - } + auto inputPathS = printInputPath(inputPath); + debug("computing input '%s'", inputPathS); - /* Go over the flake inputs, resolve/fetch them if - necessary (i.e. if they're new or the flakeref changed - from what's in the lock file). */ - for (auto & [id, input2] : flakeInputs) { - auto inputPath(inputPathPrefix); - inputPath.push_back(id); - auto inputPathS = printInputPath(inputPath); - debug("computing input '%s'", inputPathS); + try { - /* Do we have an override for this input from one of the - ancestors? */ - auto i = overrides.find(inputPath); - bool hasOverride = i != overrides.end(); - if (hasOverride) overridesUsed.insert(inputPath); - auto & input = hasOverride ? i->second : input2; + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; - /* Resolve 'follows' later (since it may refer to an input - path we haven't processed yet. */ - if (input.follows) { - InputPath target; - if (hasOverride || input.absolute) - /* 'follows' from an override is relative to the - root of the graph. */ - target = *input.follows; - else { - /* Otherwise, it's relative to the current flake. */ - target = inputPathPrefix; - for (auto & i : *input.follows) target.push_back(i); - } - debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); - node->inputs.insert_or_assign(id, target); - continue; - } + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputPath target; + if (hasOverride || input.absolute) + /* 'follows' from an override is relative to the + root of the graph. */ + target = *input.follows; + else { + /* Otherwise, it's relative to the current flake. */ + target = inputPathPrefix; + for (auto & i : *input.follows) target.push_back(i); + } + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } - assert(input.ref); + assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ - std::shared_ptr oldLock; + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr oldLock; - updatesUsed.insert(inputPath); + updatesUsed.insert(inputPath); - if (oldNode && !lockFlags.inputUpdates.count(inputPath)) - if (auto oldLock2 = get(oldNode->inputs, id)) - if (auto oldLock3 = std::get_if<0>(&*oldLock2)) - oldLock = *oldLock3; + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; - if (oldLock - && oldLock->originalRef == *input.ref - && !hasOverride) - { - debug("keeping existing input '%s'", inputPathS); + if (oldLock + && oldLock->originalRef == *input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); - /* Copy the input from the old lock since its flakeref - didn't change and there is no override from a - higher level flake. */ - auto childNode = std::make_shared( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared( + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); - node->inputs.insert_or_assign(id, childNode); + node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = - lb != lockFlags.inputUpdates.end() - && lb->size() > inputPath.size() - && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); - } else { - /* No need to fetch this flake, we can be - lazy. However there may be new overrides on the - inputs of this flake, so we need to check - those. */ - FlakeInputs fakeInputs; + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { - if (auto lockedNode = std::get_if<0>(&i.second)) { - fakeInputs.emplace(i.first, FlakeInput { - .ref = (*lockedNode)->originalRef, - .isFlake = (*lockedNode)->isFlake, - }); - } else if (auto follows = std::get_if<1>(&i.second)) { - fakeInputs.emplace(i.first, FlakeInput { - .follows = *follows, - .absolute = true - }); + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .follows = *follows, + .absolute = true + }); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputPathS); + + if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared( + inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared(lockedRef, *input.ref, false)); } } - computeLocks(fakeInputs, childNode, inputPath, oldLock); - } - - } else { - /* We need to create a new lock file entry. So fetch - this input. */ - debug("creating new input '%s'", inputPathS); - - if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) - throw Error("cannot update flake input '%s' in pure mode", inputPathS); - - if (input.isFlake) { - auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); - - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto childNode = std::make_shared( - inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); - - node->inputs.insert_or_assign(id, childNode); - - /* Guard against circular flake imports. */ - for (auto & parent : parents) - if (parent == *input.ref) - throw Error("found circular import of flake '%s'", parent); - parents.push_back(*input.ref); - Finally cleanup([&]() { parents.pop_back(); }); - - /* Recursively process the inputs of this - flake. Also, unless we already have this flake - in the top-level lock file, use this flake's - own lock file. */ - computeLocks( - inputFlake.inputs, childNode, inputPath, - oldLock - ? std::dynamic_pointer_cast(oldLock) - : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); - } - - else { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, lockFlags.useRegistries, flakeCache); - node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, *input.ref, false)); + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", inputPathS); + throw; } } - } - }; + }; - computeLocks( - flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); - for (auto & i : lockFlags.inputOverrides) - if (!overridesUsed.count(i.first)) - warn("the flag '--override-input %s %s' does not match any input", - printInputPath(i.first), i.second); + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + printInputPath(i.first), i.second); - for (auto & i : lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", printInputPath(i)); - /* Check 'follows' inputs. */ - newLockFile.check(); + /* Check 'follows' inputs. */ + newLockFile.check(); - debug("new lock file: %s", newLockFile); + debug("new lock file: %s", newLockFile); - /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { - auto diff = LockFile::diff(oldLockFile, newLockFile); + auto diff = LockFile::diff(oldLockFile, newLockFile); - if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { - if (!newLockFile.isImmutable()) { - if (settings.warnDirty) - warn("will not write lock file of flake '%s' because it has a mutable input", topRef); - } else { - if (!lockFlags.updateLockFile) - throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input.getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto path = *sourcePath + "/" + relPath; + auto path = *sourcePath + "/" + relPath; - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(path); - if (lockFileExists) { - auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", path); - else - warn("updating lock file '%s':\n%s", path, s); - } else - warn("creating lock file '%s'", path); + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", path); + else + warn("updating lock file '%s':\n%s", path, s); + } else + warn("creating lock file '%s'", path); - newLockFile.write(path); + newLockFile.write(path); - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - lockFlags.commitLockFile - ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", - relPath, lockFileExists ? "Update" : "Add", diff)) - : std::nullopt); + topRef.input.markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); - /* Rewriting the lockfile changed the top-level - repo, so we should re-read it. FIXME: we could - also just clear the 'rev' field... */ - auto prevLockedRef = flake.lockedRef; - FlakeCache dummyCache; - flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); - if (lockFlags.commitLockFile && - flake.lockedRef.input.getRev() && - prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) - warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + if (lockFlags.commitLockFile && + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); - /* Make sure that we picked up the change, - i.e. the tree should usually be dirty - now. Corner case: we could have reverted from a - dirty to a clean tree! */ - if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input.isImmutable()) - throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); - } + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input.isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); } else - throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); - } else - warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); - } + warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + } - return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + + } catch (Error & e) { + e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); + throw; + } } void callFlake(EvalState & state, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e6741a451..916e0a8e8 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -132,7 +132,14 @@ std::pair Input::fetch(ref store) const } } - auto [tree, input] = scheme->fetch(store, *this); + auto [tree, input] = [&]() -> std::pair { + try { + return scheme->fetch(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } + }(); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); From 965dc6070a1b7dc582d90039c670d436f4a2e9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:04:49 +0100 Subject: [PATCH 334/477] Drop trailing whitespace --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 563f49170..8ea5cdc9d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -856,7 +856,7 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); else err.msg = hf; } From 12de0466fea6558ccb0dd5b98b72d7a068c9b5e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:46:10 +0100 Subject: [PATCH 335/477] Add trace to build errors during import-from-derivation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example: error: builder for '/nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv' failed with exit code 1; last 1 log lines: > FAIL For full logs, run 'nix log /nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv'. … while importing '/nix/store/pfp4a4bjh642ylxyipncqs03z6kkgfvy-failing' at /nix/store/25wgzr2qrqqiqfbdb1chpiry221cjglc-source/flake.nix:58:15: 57| 58| ifd = import self.hydraJobs.broken; | ^ 59| --- src/libexpr/primops.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a470ed6df..13565b950 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -118,6 +118,9 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); + } catch (Error & e) { + e.addTrace(pos, "while importing '%s'", path); + throw e; } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); From 9355ecd54301372b6a919a2205340f904c7a51c6 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 17:24:30 +0100 Subject: [PATCH 336/477] Add a new Cmd type working on RealisedPaths Where a `RealisedPath` is a store path with its history, meaning either an opaque path for stuff that has been directly added to the store, or a `Realisation` for stuff that has been built by a derivation This is a low-level refactoring that doesn't bring anything by itself (except a few dozen extra lines of code :/ ), but raising the abstraction level a bit is important on a number of levels: - Commands like `nix build` have to query for the realisations after the build is finished which is fragile (see 27905f12e4a7207450abe37c9ed78e31603b67e1 for example). Having them oprate directly at the realisation level would avoid that - Others like `nix copy` currently operate directly on (built) store paths, but need a bit more information as they will need to register the realisations on the remote side --- src/libcmd/command.cc | 42 ++++++++++------- src/libcmd/command.hh | 23 ++++++++-- src/libcmd/installables.cc | 48 ++++++++++++++++---- src/libstore/realisation.cc | 31 +++++++++++++ src/libstore/realisation.hh | 90 +++++++++++++++++++++++++++++++++---- src/nix/copy.cc | 2 + 6 files changed, 200 insertions(+), 36 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 614dee788..efdc98d5a 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,7 +54,7 @@ void StoreCommand::run() run(getStore()); } -StorePathsCommand::StorePathsCommand(bool recursive) +RealisedPathsCommand::RealisedPathsCommand(bool recursive) : recursive(recursive) { if (recursive) @@ -81,30 +81,40 @@ StorePathsCommand::StorePathsCommand(bool recursive) }); } -void StorePathsCommand::run(ref store) +void RealisedPathsCommand::run(ref store) { - StorePaths storePaths; - + std::vector paths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); + // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); - } - - else { - for (auto & p : toStorePaths(store, realiseMode, operateOn, installables)) - storePaths.push_back(p); - + paths.push_back(p); + } else { + auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables); if (recursive) { - StorePathSet closure; - store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); - storePaths.clear(); - for (auto & p : closure) - storePaths.push_back(p); + auto roots = std::move(pathSet); + pathSet = {}; + RealisedPath::closure(*store, roots, pathSet); } + for (auto & path : pathSet) + paths.push_back(path); } + run(store, std::move(paths)); +} + +StorePathsCommand::StorePathsCommand(bool recursive) + : RealisedPathsCommand(recursive) +{ +} + +void StorePathsCommand::run(ref store, std::vector paths) +{ + StorePaths storePaths; + for (auto & p : paths) + storePaths.push_back(p.path()); + run(store, std::move(storePaths)); } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index ed6980075..8c0b3a94a 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -141,7 +141,7 @@ private: }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public InstallablesCommand +struct RealisedPathsCommand : public InstallablesCommand { private: @@ -154,17 +154,28 @@ protected: public: - StorePathsCommand(bool recursive = false); + RealisedPathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, std::vector storePaths) = 0; + virtual void run(ref store, std::vector paths) = 0; void run(ref store) override; bool useDefaultInstallables() override { return !all; } }; +struct StorePathsCommand : public RealisedPathsCommand +{ + StorePathsCommand(bool recursive = false); + + using RealisedPathsCommand::run; + + virtual void run(ref store, std::vector storePaths) = 0; + + void run(ref store, std::vector paths) override; +}; + /* A command that operates on exactly one store path. */ struct StorePathCommand : public InstallablesCommand { @@ -218,6 +229,12 @@ std::set toDerivations(ref store, std::vector> installables, bool useDeriver = false); +std::set toRealisedPaths( + ref store, + Realise mode, + OperateOn operateOn, + std::vector> installables); + /* Helper function to generate args that invoke $EDITOR on filename:lineno. */ Strings editorFor(const Pos & pos); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4e6bf4a9a..98a27ded9 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -704,23 +704,43 @@ Buildables build(ref store, Realise mode, return buildables; } -StorePathSet toStorePaths(ref store, - Realise mode, OperateOn operateOn, +std::set toRealisedPaths( + ref store, + Realise mode, + OperateOn operateOn, std::vector> installables) { - StorePathSet outPaths; - + std::set res; if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { [&](BuildableOpaque bo) { - outPaths.insert(bo.path); + res.insert(bo.path); }, [&](BuildableFromDrv bfd) { + auto drv = store->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { - if (!output.second) - throw Error("Cannot operate on output of unbuilt CA drv"); - outPaths.insert(*output.second); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (!outputHashes.count(output.first)) + throw Error( + "The derivation %s doesn't have an output " + "named %s", + store->printStorePath(bfd.drvPath), + output.first); + auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string()); + res.insert(RealisedPath{*realisation}); + } + else { + // If ca-derivations isn't enabled, behave as if + // all the paths are opaque to keep the default + // behavior + assert(output.second); + res.insert(*output.second); + } } }, }, b); @@ -731,9 +751,19 @@ StorePathSet toStorePaths(ref store, for (auto & i : installables) for (auto & b : i->toBuildables()) if (auto bfd = std::get_if(&b)) - outPaths.insert(bfd->drvPath); + res.insert(bfd->drvPath); } + return res; +} + +StorePathSet toStorePaths(ref store, + Realise mode, OperateOn operateOn, + std::vector> installables) +{ + StorePathSet outPaths; + for (auto & path : toRealisedPaths(store, mode, operateOn, installables)) + outPaths.insert(path.path()); return outPaths; } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47ad90eee..c9b66186f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -46,4 +46,35 @@ Realisation Realisation::fromJSON( }; } +StorePath RealisedPath::path() const { + return visit([](auto && arg) { return arg.getPath(); }); +} + +void RealisedPath::closure( + Store& store, + const RealisedPath::Set& startPaths, + RealisedPath::Set& ret) +{ + // FIXME: This only builds the store-path closure, not the real realisation + // closure + StorePathSet initialStorePaths, pathsClosure; + for (auto& path : startPaths) + initialStorePaths.insert(path.path()); + store.computeFSClosure(initialStorePaths, pathsClosure); + ret.insert(startPaths.begin(), startPaths.end()); + ret.insert(pathsClosure.begin(), pathsClosure.end()); +} + +void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +{ + RealisedPath::closure(store, {*this}, ret); +} + +RealisedPath::Set RealisedPath::closure(Store& store) const +{ + RealisedPath::Set ret; + closure(store, ret); + return ret; +} + } // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 4b8ead3c5..1ecddc4d1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,34 @@ #include "path.hh" #include + +/* Awfull hacky generation of the comparison operators by doing a lexicographic + * comparison between the choosen fields + * ``` + * GENERATE_CMP(ClassName, my->field1, my->field2, ...) + * ``` + * + * will generate comparison operators semantically equivalent to: + * ``` + * bool operator<(const ClassName& other) { + * return field1 < other.field1 && field2 < other.field2 && ...; + * } + * ``` + */ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ + bool operator COMPARATOR(const MY_TYPE& other) const { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple( FIELDS ); \ + me = &other; \ + auto fields2 = std::make_tuple( FIELDS ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) +#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_CMP(args...) \ + GENERATE_EQUAL(args) \ + GENERATE_LEQ(args) + namespace nix { struct DrvOutput { @@ -17,13 +45,7 @@ struct DrvOutput { static DrvOutput parse(const std::string &); - bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } - bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } - -private: - // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvHash, outputName); } + GENERATE_CMP(DrvOutput, me->drvHash, me->outputName); }; struct Realisation { @@ -32,8 +54,60 @@ struct Realisation { nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + + StorePath getPath() const { return outPath; } + + GENERATE_CMP(Realisation, me->id, me->outPath); }; -typedef std::map DrvOutputs; +struct OpaquePath { + StorePath path; + + StorePath getPath() const { return path; } + + GENERATE_CMP(OpaquePath, me->path); +}; + + +/** + * A store path with all the history of how it went into the store + */ +struct RealisedPath { + /* + * A path is either the result of the realisation of a derivation or + * an opaque blob that has been directly added to the store + */ + using Raw = std::variant; + Raw raw; + + using Set = std::set; + + RealisedPath(StorePath path) : raw(OpaquePath{path}) {} + RealisedPath(Realisation r) : raw(r) {} + + /** + * Syntactic sugar to run `std::visit` on the raw value: + * path.visit(blah) == std::visit(blah, path.raw) + */ + template + constexpr decltype(auto) visit(Visitor && vis) { + return std::visit(vis, raw); + } + template + constexpr decltype(auto) visit(Visitor && vis) const { + return std::visit(vis, raw); + } + + /** + * Get the raw store path associated to this + */ + StorePath path() const; + + void closure(Store& store, Set& ret) const; + static void closure(Store& store, const Set& startPaths, Set& ret); + Set closure(Store& store) const; + + GENERATE_CMP(RealisedPath, me->raw); +}; } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f15031a45..c56a1def1 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -16,6 +16,8 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; + using StorePathsCommand::run; + CmdCopy() : StorePathsCommand(true) { From 991edaace57d50d571f4f4658ef2d52b94a07f2c Mon Sep 17 00:00:00 2001 From: James Ottaway Date: Fri, 29 Jan 2021 13:55:18 +1000 Subject: [PATCH 337/477] Shorten `mktemp` flag for macOS Address `mktemp: illegal option -- -`. --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 578258394..3c44fdb0e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -239,7 +239,7 @@ struct Common : InstallableCommand, MixProfile out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); From d5acc4865c8a5853bc5ede606d98c8055f8afdb2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 29 Jan 2021 18:31:40 +0100 Subject: [PATCH 338/477] Use passthru for perl-bindings, allows Nix patching for Hydra This allows patching Nix for Hydra with additional overlays, because `.overrideAttrs` and co. will persist the passthru's --- flake.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 9addccd63..830cceb9f 100644 --- a/flake.nix +++ b/flake.nix @@ -115,7 +115,7 @@ # 'nix.perl-bindings' packages. overlay = final: prev: { - nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -163,9 +163,8 @@ installCheckFlags = "sysconfdir=$(out)/etc"; separateDebugInfo = true; - }) // { - perl-bindings = with final; stdenv.mkDerivation { + passthru.perl-bindings = with final; stdenv.mkDerivation { name = "nix-perl-${version}"; src = self; From d0b74e2d2506b9237263ad1294eb7297c99a5e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 1 Feb 2021 13:11:42 +0000 Subject: [PATCH 339/477] --no-net -> --offline --- src/nix/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 58b643cc5..e95b04d85 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -91,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs }); addFlag({ - .longName = "no-net", + .longName = "offline", .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); From fb00e7dc529f54e6b2d864532e93ef3645b1b704 Mon Sep 17 00:00:00 2001 From: Dominik Schrempf Date: Mon, 1 Feb 2021 17:42:14 +0100 Subject: [PATCH 340/477] Remove newline in operator table. --- doc/manual/src/expressions/language-operators.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index 1d787ffe3..b7fd6f4c6 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -25,5 +25,4 @@ order of precedence (from strongest to weakest binding). | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | | Logical OR | *e1* `\|\|` *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| - e2`). | 14 | +| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| e2`). | 14 | From 3d1bbabe55eff6e67d91e0cbee781c2b756a2e92 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 2 Feb 2021 19:50:03 -0600 Subject: [PATCH 341/477] Use derivation output name from toDerivation This fixes an issue where derivations with a primary output that is not "out" would fail with: $ nix profile install nixpkgs#sqlite error: opening directory '/nix/store/2a2ydlgyydly5czcc8lg12n6qqkfz863-sqlite-3.34.1-bin': No such file or directory This happens because while derivations produce every output when built, you might not have them if you didn't build the derivation yourself (for instance, the store path was fetch from a binary cache). This uses outputName provided from DerivationInfo which appears to match the first output of the derivation. --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 765d6866e..827f8be5a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { From 76d8bdfe355aa1976580f4fa8f11f1ec505a6c66 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 2 Feb 2021 23:04:36 +0100 Subject: [PATCH 342/477] Include note about type of catched errors in tryEval documentation Reference #356. --- src/libexpr/primops.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 13565b950..1d1afa768 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -696,10 +696,14 @@ static RegisterPrimOp primop_tryEval({ Try to shallowly evaluate *e*. Return a set containing the attributes `success` (`true` if *e* evaluated successfully, `false` if an error was thrown) and `value`, equalling *e* if - successful and `false` otherwise. Note that this doesn't evaluate - *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval - e).success ` will be `true`. Using ` builtins.deepSeq ` one can - get the expected result: `let e = { x = throw ""; }; in + successful and `false` otherwise. `tryEval` will only prevent + errors created by `throw` or `assert` from being thrown. + Errors `tryEval` will not catch are for example those created + by `abort` and type errors generated by builtins. Also note that + this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; + in (builtins.tryEval e).success` will be `true`. Using + `builtins.deepSeq` one can get the expected result: + `let e = { x = throw ""; }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be `false`. )", From e38cd5becbbff57951b6a576dd793f4777a9833c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Feb 2021 21:22:11 -0600 Subject: [PATCH 343/477] Always enter first level of attrset in nix search This makes nix search always go through the first level of an attribute set, even if it's not a top level attribute. For instance, you can now list all GHC compilers with: $ nix search nixpkgs#haskell.compiler ... This is similar to how nix-env works when you pass in -A. --- src/nix/search.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 9f864b3a4..c52a48d4e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -81,9 +81,9 @@ struct CmdSearch : InstallableCommand, MixJSON uint64_t results = 0; - std::function & attrPath)> visit; + std::function & attrPath, bool initialRecurse)> visit; - visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath) + visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath, bool initialRecurse) { Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); @@ -94,7 +94,7 @@ struct CmdSearch : InstallableCommand, MixJSON auto cursor2 = cursor.getAttr(attr); auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*cursor2, attrPath2); + visit(*cursor2, attrPath2, false); } }; @@ -150,6 +150,9 @@ struct CmdSearch : InstallableCommand, MixJSON || (attrPath[0] == "packages" && attrPath.size() <= 2)) recurse(); + else if (initialRecurse) + recurse(); + else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); if (attr && attr->getBool()) @@ -163,7 +166,7 @@ struct CmdSearch : InstallableCommand, MixJSON }; for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix)); + visit(*cursor, parseAttrPath(*state, prefix), true); if (!json && !results) throw Error("no results for the given search term(s)!"); From ca8facefb6b6b0ffd6e22507111847dbfc9a3c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 14:47:28 +0100 Subject: [PATCH 344/477] Normalize some error messages Co-authored-by: Eelco Dolstra --- src/libcmd/installables.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 98a27ded9..9ad02b5f0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -724,14 +724,13 @@ std::set toRealisedPaths( if (settings.isExperimentalFeatureEnabled("ca-derivations")) { if (!outputHashes.count(output.first)) throw Error( - "The derivation %s doesn't have an output " - "named %s", + "the derivation '%s' doesn't have an output named '%s'", store->printStorePath(bfd.drvPath), output.first); auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; auto realisation = store->queryRealisation(outputId); if (!realisation) - throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string()); + throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string()); res.insert(RealisedPath{*realisation}); } else { From 43d409f6690b79b5d4e1ab5e9780de93eb0f677a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 14:47:56 +0100 Subject: [PATCH 345/477] Fix a whitespace issue Co-authored-by: Eelco Dolstra --- src/libstore/realisation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index c9b66186f..e4276c040 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -65,7 +65,7 @@ void RealisedPath::closure( ret.insert(pathsClosure.begin(), pathsClosure.end()); } -void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const { RealisedPath::closure(store, {*this}, ret); } From d2091af231ab97b729c2486b55e520c565e59dd3 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:11:05 +0100 Subject: [PATCH 346/477] Move the GENERATE_CMP macro to its own file Despite being an ugly hack, it can probably be useful in a couple extra places --- src/libstore/realisation.hh | 29 +---------------------------- src/libutil/comparator.hh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 src/libutil/comparator.hh diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 1ecddc4d1..557f54362 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -2,34 +2,7 @@ #include "path.hh" #include - - -/* Awfull hacky generation of the comparison operators by doing a lexicographic - * comparison between the choosen fields - * ``` - * GENERATE_CMP(ClassName, my->field1, my->field2, ...) - * ``` - * - * will generate comparison operators semantically equivalent to: - * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; - * } - * ``` - */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) +#include "comparator.hh" namespace nix { diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh new file mode 100644 index 000000000..0315dc506 --- /dev/null +++ b/src/libutil/comparator.hh @@ -0,0 +1,30 @@ +#pragma once + +/* Awfull hacky generation of the comparison operators by doing a lexicographic + * comparison between the choosen fields. + * + * ``` + * GENERATE_CMP(ClassName, me->field1, me->field2, ...) + * ``` + * + * will generate comparison operators semantically equivalent to: + * + * ``` + * bool operator<(const ClassName& other) { + * return field1 < other.field1 && field2 < other.field2 && ...; + * } + * ``` + */ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ + bool operator COMPARATOR(const MY_TYPE& other) const { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple( FIELDS ); \ + me = &other; \ + auto fields2 = std::make_tuple( FIELDS ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) +#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_CMP(args...) \ + GENERATE_EQUAL(args) \ + GENERATE_LEQ(args) From e69cfdebb090b3aabbff69a44504883d5b6fb866 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:15:22 +0100 Subject: [PATCH 347/477] Remove the `visit` machinery in `RealisedPath` In addition to being some ugly template trickery, it was also totally useless as it was used in only one place where I could replace it by just a few extra characters --- src/libstore/realisation.cc | 2 +- src/libstore/realisation.hh | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index e4276c040..cd74af4ee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -47,7 +47,7 @@ Realisation Realisation::fromJSON( } StorePath RealisedPath::path() const { - return visit([](auto && arg) { return arg.getPath(); }); + return std::visit([](auto && arg) { return arg.getPath(); }, raw); } void RealisedPath::closure( diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 557f54362..7c91d802a 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -58,19 +58,6 @@ struct RealisedPath { RealisedPath(StorePath path) : raw(OpaquePath{path}) {} RealisedPath(Realisation r) : raw(r) {} - /** - * Syntactic sugar to run `std::visit` on the raw value: - * path.visit(blah) == std::visit(blah, path.raw) - */ - template - constexpr decltype(auto) visit(Visitor && vis) { - return std::visit(vis, raw); - } - template - constexpr decltype(auto) visit(Visitor && vis) const { - return std::visit(vis, raw); - } - /** * Get the raw store path associated to this */ From 0187838e2e7ff01f1b480e3e85d9e96da0b4b78e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 12:11:50 +0100 Subject: [PATCH 348/477] Add a trace to readLine() failures Hopefully this helps to diagnose 'error: unexpected EOF reading a line' on macOS. --- src/libstore/build/derivation-goal.cc | 25 ++++++++++++++++++++++--- tests/init.sh | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8717499c0..190adf31c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1044,7 +1044,14 @@ HookReply DerivationGoal::tryBuildHook() whether the hook wishes to perform the build. */ string reply; while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); + auto s = [&]() { + try { + return readLine(worker.hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response from the build hook"); + throw e; + } + }(); if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) ; else if (string(s, 0, 2) == "# ") { @@ -1084,7 +1091,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); - machineName = readLine(hook->fromHook.readSide.get()); + try { + machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw e; + } /* Tell the hook all the inputs that have to be copied to the remote system. */ @@ -1773,7 +1785,14 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { - string msg = readLine(builderOut.readSide.get()); + string msg = [&]() { + try { + return readLine(builderOut.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response of setting up the build environment"); + throw e; + } + }(); if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { FdSource source(builderOut.readSide.get()); diff --git a/tests/init.sh b/tests/init.sh index 63cf895e2..1a6ccb6fe 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -21,6 +21,7 @@ experimental-features = nix-command flakes gc-reserved-space = 0 substituters = flake-registry = $TEST_ROOT/registry.json +show-trace = true include nix.conf.extra EOF From 480426a364f09e7992230b32f2941a09fb52d729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 15:57:33 +0100 Subject: [PATCH 349/477] Add more instrumentation for #4270 --- src/libstore/build/derivation-goal.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 190adf31c..eeaec4f2c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1784,12 +1784,14 @@ void DerivationGoal::startBuilder() worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); /* Check if setting up the build environment failed. */ + std::vector msgs; while (true) { string msg = [&]() { try { return readLine(builderOut.readSide.get()); } catch (Error & e) { - e.addTrace({}, "while reading the response of setting up the build environment"); + e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + concatStringsSep("|", msgs)); throw e; } }(); @@ -1801,6 +1803,7 @@ void DerivationGoal::startBuilder() throw ex; } debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); } } From d0e34c85f85510cb2ef591de29693b4cf8bdc65b Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Sat, 6 Feb 2021 12:59:11 +0100 Subject: [PATCH 350/477] libcmd/markdown: handle allocation errors in lowdown_term_rndr We upgrade to lowdown 0.8.0 [1] which contains a fix/improvement to a behavior mentioned in this issue thread [2] where a big part of lowdown's API would just call exit(1) on allocation errors since that is a satisfying behavior for the lowdown binary. Now lowdown_term_rndr returns 0 if an allocation error occurred which we check for in libcmd/markdown.cc. Also the extern "C" { } wrapper around lowdown.h has been removed as it is not necessary. [1]: https://github.com/kristapsdz/lowdown/blob/6ca7c855a063d1c77ae0b89405047cc3913a74d8/versions.xml#L987-L1006 [2]: https://github.com/kristapsdz/lowdown/issues/45#issuecomment-756681153 --- flake.nix | 8 ++++---- src/libcmd/markdown.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index d94da9dae..8c60934e6 100644 --- a/flake.nix +++ b/flake.nix @@ -198,12 +198,12 @@ }; - lowdown = with final; stdenv.mkDerivation { - name = "lowdown-0.7.9"; + lowdown = with final; stdenv.mkDerivation rec { + name = "lowdown-0.8.0"; src = fetchurl { - url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz; - hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ=="; + url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz"; + hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw=="; }; #src = lowdown-src; diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 40788a42f..d25113d93 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -3,9 +3,7 @@ #include "finally.hh" #include -extern "C" { #include -} namespace nix { @@ -42,7 +40,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("cannot allocate Markdown output buffer"); Finally freeBuffer([&]() { lowdown_buf_free(buf); }); - lowdown_term_rndr(buf, nullptr, renderer, node); + int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node); + if (!rndr_res) + throw Error("allocation error while rendering Markdown"); return std::string(buf->data, buf->size); } From 6af26b7aec28e8bf1786ead3ba26beb50317c167 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Sat, 6 Feb 2021 13:29:38 +0100 Subject: [PATCH 351/477] Add Stale bot The configuration was taken from nixpkgs repository and adjusted to `NixOS/nix`. A `stale` label was added to the labels (with gray color). Issues and PRs with `critical` label are excluded from interacting with the stale bot. --- .github/STALE-BOT.md | 35 +++++++++++++++++++++++++++++++++++ .github/stale.yml | 9 +++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/STALE-BOT.md create mode 100644 .github/stale.yml diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md new file mode 100644 index 000000000..6cc03f540 --- /dev/null +++ b/.github/STALE-BOT.md @@ -0,0 +1,35 @@ +# Stale bot information + +- Thanks for your contribution! +- To remove the stale label, just leave a new comment. +- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos). + +## Suggestions for PRs + +1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else. +2. If it is unfinished but you plan to finish it, please mark it as a draft. +3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. +4. To get things rolling again, rebase the PR against the target branch and address valid comments. +5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604). +6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions. + +## Suggestions for issues + +1. If it is resolved (either for you personally, or in general), please consider closing it. +2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough. +3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/). +4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude! + +**Memorandum on closing issues** + +Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort. + +## Useful GitHub search queries + +- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) +- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) +- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+) +- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..f81b4c762 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,9 @@ +# Configuration for probot-stale - https://github.com/probot/stale +daysUntilStale: 180 +daysUntilClose: false +exemptLabels: + - "critical" +staleLabel: "2.status: stale" +markComment: | + I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) +closeComment: false From 91d83426f70bbf28c1bf92be5f662d76d1d47578 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Sat, 6 Feb 2021 13:33:34 +0100 Subject: [PATCH 352/477] typo --- .github/STALE-BOT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 6cc03f540..5e8f5d929 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -27,7 +27,7 @@ Don't be afraid to close an issue that holds valuable information. Closed issues ## Useful GitHub search queries -- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) - [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) - [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) - [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) From 7c112351d9e941567e64063638396259546d9a48 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Sun, 7 Feb 2021 13:56:50 +0000 Subject: [PATCH 353/477] libutil: EPERM from kill(-1, ...) is fine I tested a trivial program that called kill(-1, SIGKILL), which was run as the only process for an unpriveleged user, on Linux and FreeBSD. On Linux, kill reported success, while on FreeBSD it failed with EPERM. POSIX says: > If pid is -1, sig shall be sent to all processes (excluding an > unspecified set of system processes) for which the process has > permission to send that signal. and > The kill() function is successful if the process has permission to > send sig to any of the processes specified by pid. If kill() fails, > no signal shall be sent. and > [EPERM] > The process does not have permission to send the signal to any > receiving process. My reading of this is that kill(-1, ...) may fail with EPERM when there are no other processes to kill (since the current process is ignored). Since kill(-1, ...) only attempts to kill processes the user has permission to kill, it can't mean that we tried to do something we didn't have permission to kill, so it should be fine to interpret EPERM the same as success here for any POSIX-compliant system. This fixes an issue that Mic92 encountered[1] when he tried to review a Nixpkgs PR on FreeBSD. [1]: https://github.com/NixOS/nixpkgs/pull/81459#issuecomment-606073668 --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 89f7b58f8..ef37275ac 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -946,7 +946,7 @@ void killUser(uid_t uid) #else if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ if (errno != EINTR) throw SysError("cannot kill processes for uid '%1%'", uid); } From 37352aa7e19e0bfebbd0c32985cbf79a83508538 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 7 Feb 2021 20:44:56 +0100 Subject: [PATCH 354/477] Support --no-net for backwards compatibility --- src/libutil/args.cc | 3 +++ src/libutil/args.hh | 1 + src/nix/main.cc | 1 + 3 files changed, 5 insertions(+) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 71bae0504..9377fe4c0 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -14,6 +14,8 @@ void Args::addFlag(Flag && flag_) assert(flag->handler.arity == flag->labels.size()); assert(flag->longName != ""); longFlags[flag->longName] = flag; + for (auto & alias : flag->aliases) + longFlags[alias] = flag; if (flag->shortName) shortFlags[flag->shortName] = flag; } @@ -191,6 +193,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 42d8515ef..88f068087 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -97,6 +97,7 @@ protected: typedef std::shared_ptr ptr; std::string longName; + std::set aliases; char shortName = 0; std::string description; std::string category; diff --git a/src/nix/main.cc b/src/nix/main.cc index e95b04d85..ef5e41a55 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "offline", + .aliases = {"no-net"}, // FIXME: remove .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); From bab3f30755490207446966e9e828119462b57141 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Mon, 8 Feb 2021 11:49:07 +0100 Subject: [PATCH 355/477] Auto closing issues/PRs after 1year. --- .github/stale.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index f81b4c762..fe24942f4 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,9 +1,10 @@ # Configuration for probot-stale - https://github.com/probot/stale daysUntilStale: 180 -daysUntilClose: false +daysUntilClose: 365 exemptLabels: - "critical" -staleLabel: "2.status: stale" +staleLabel: "stale" markComment: | I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) -closeComment: false +closeComment: | + I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) From f2245091d033a8037aeb29ae701d20611500af6d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Feb 2021 12:26:41 -0500 Subject: [PATCH 356/477] Revert "narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash" --- src/libstore/binary-cache-store.cc | 6 +++++- tests/binary-cache.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 15163ead5..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,7 +176,11 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index f8d47170f..6697ce236 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -60,7 +60,7 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar | head -n1) +nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar From ad337c8697099ac9deb6e0ac16ea91d8acc51e4f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Feb 2021 17:33:28 +0000 Subject: [PATCH 357/477] Deeper `Command` hierarchy to remove redundancy Simply put, we now have `StorePathCommand : public StorePathsCommand` so `StorePathCommand` doesn't reimplement work. --- src/libcmd/command.cc | 4 +--- src/libcmd/command.hh | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index efdc98d5a..d29954f67 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -118,10 +118,8 @@ void StorePathsCommand::run(ref store, std::vector paths) run(store, std::move(storePaths)); } -void StorePathCommand::run(ref store) +void StorePathCommand::run(ref store, std::vector storePaths) { - auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables); - if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8c0b3a94a..c02193924 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -177,13 +177,13 @@ struct StorePathsCommand : public RealisedPathsCommand }; /* A command that operates on exactly one store path. */ -struct StorePathCommand : public InstallablesCommand +struct StorePathCommand : public StorePathsCommand { - using StoreCommand::run; + using StorePathsCommand::run; virtual void run(ref store, const StorePath & storePath) = 0; - void run(ref store) override; + void run(ref store, std::vector storePaths) override; }; /* A helper class for registering commands globally. */ From 35129884f9348f068d538e67bb559cc6104f714e Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Tue, 16 Feb 2021 23:19:42 +0000 Subject: [PATCH 358/477] Fix Haskell example http://nixos.org redirects to https://nixos.org and apparently the HTTP library doesn't follow the redirect, so the output is empty. When defining https in the request it crashes because the library doesn't seem to support https. So this switches the example to a different http library. --- doc/manual/src/command-ref/nix-shell.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 88b675e71..938d56e6e 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -232,22 +232,23 @@ terraform apply > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following -Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable +Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz +#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz -import Network.HTTP +import Network.Curl.Download import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) -- Fetch nixos.org and print all hrefs. main = do - resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/") - body <- getResponseBody resp - let tags = filter (isTagOpenName "a") $ parseTags body + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp let tags' = map (fromAttrib "href") tags mapM_ putStrLn $ filter (/= "") tags' ``` From 5f4701e70d35bb9ea2fb659caf387a30001e28ce Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Tue, 16 Feb 2021 23:27:04 +0000 Subject: [PATCH 359/477] Update doc/manual/src/command-ref/nix-shell.md Co-authored-by: Cole Helbling --- doc/manual/src/command-ref/nix-shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 938d56e6e..54812a49f 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -238,7 +238,7 @@ branch): ```haskell #! /usr/bin/env nix-shell #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download import Text.HTML.TagSoup From 6042febfce3011aaa5e3c369ea14a0d93ad2880e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 15:30:49 +0100 Subject: [PATCH 360/477] Restore warning about 'nix' being experimental Fixes #4552. --- doc/manual/generate-manpage.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index a563c31f8..964b57086 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -7,7 +7,10 @@ let showCommand = { command, def, filename }: - "# Name\n\n" + '' + **Warning**: This program is **experimental** and its interface is subject to change. + '' + + "# Name\n\n" + "`${command}` - ${def.description}\n\n" + "# Synopsis\n\n" + showSynopsis { inherit command; args = def.args; } From 063de66909ff1b20394cdebdca1ef62bb6ca1d51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 16:42:03 +0100 Subject: [PATCH 361/477] nix develop: Fix quoted string handling Fixes #4540. --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3c44fdb0e..0938cbe5b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -59,7 +59,7 @@ BuildEnvironment readEnvironment(const Path & path) R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; static std::string squotedStringRegex = - R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; static std::string indexedArrayRegex = R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; From cced73496b835b545be91cbebc4f89f61a7b106f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 16:53:19 +0100 Subject: [PATCH 362/477] nix flake show: Handle 'overlays' output Fixes #4542. --- src/nix/flake.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4cd7d77a0..091af8084 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -880,7 +880,8 @@ struct CmdFlakeShow : FlakeCommand || attrPath[0] == "nixosConfigurations" || attrPath[0] == "nixosModules" || attrPath[0] == "defaultApp" - || attrPath[0] == "templates")) + || attrPath[0] == "templates" + || attrPath[0] == "overlays")) || ((attrPath.size() == 1 || attrPath.size() == 2) && (attrPath[0] == "checks" || attrPath[0] == "packages" @@ -943,7 +944,8 @@ struct CmdFlakeShow : FlakeCommand else { logger->cout("%s: %s", headerPrefix, - attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : + (attrPath.size() == 1 && attrPath[0] == "overlay") + || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" : ANSI_YELLOW "unknown" ANSI_NORMAL); From f33878b6562c746d5865a86e64f02c75feaf5b3e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:11:14 +0100 Subject: [PATCH 363/477] Make 'nix --version -vv' work Fixes #3743. --- src/nix/main.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index ef5e41a55..5f4eb8918 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -61,6 +61,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs bool printBuildLogs = false; bool useNet = true; bool refresh = false; + bool showVersion = false; NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { @@ -87,7 +88,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "version", .description = "Show version information.", - .handler = {[&]() { if (!completions) printVersion(programName); }}, + .handler = {[&]() { showVersion = true; }}, }); addFlag({ @@ -280,6 +281,11 @@ void mainWrapped(int argc, char * * argv) initPlugins(); + if (args.showVersion) { + printVersion(programName); + return; + } + if (!args.command) throw UsageError("no subcommand specified"); From 13897afbe6cf7ef8013c0c94109696bb7b13d0c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:32:10 +0100 Subject: [PATCH 364/477] Throw an error if --arg / --argstr is used with a flake Fixes #3949. --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++-- src/libcmd/installables.hh | 12 +++++++----- src/nix/bundle.cc | 2 +- src/nix/develop.cc | 1 + src/nix/flake.cc | 2 +- src/nix/profile.cc | 8 +++++++- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9ad02b5f0..4739dc974 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -496,6 +496,23 @@ static std::string showAttrPaths(const std::vector & paths) return s; } +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + Strings && attrPaths, + Strings && prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(attrPaths), + prefixes(prefixes), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + std::tuple InstallableFlake::toDerivation() { auto lockedFlake = getLockedFlake(); @@ -628,9 +645,12 @@ std::vector> SourceExprCommand::parseInstallables( try { auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); result.push_back(std::make_shared( - getEvalState(), std::move(flakeRef), + this, + getEvalState(), + std::move(flakeRef), fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment}, - getDefaultFlakeAttrPathPrefixes(), lockFlags)); + getDefaultFlakeAttrPathPrefixes(), + lockFlags)); continue; } catch (...) { ex = std::current_exception(); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index f37b3f829..b714f097b 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -104,11 +104,13 @@ struct InstallableFlake : InstallableValue const flake::LockFlags & lockFlags; mutable std::shared_ptr _lockedFlake; - InstallableFlake(ref state, FlakeRef && flakeRef, - Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags) - : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths), - prefixes(prefixes), lockFlags(lockFlags) - { } + InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + Strings && attrPaths, + Strings && prefixes, + const flake::LockFlags & lockFlags); std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 1789e4598..48f4eb6e3 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -74,7 +74,7 @@ struct CmdBundle : InstallableCommand auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; - auto bundler = InstallableFlake( + auto bundler = InstallableFlake(this, evalState, std::move(bundlerFlakeRef), Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, Strings({"bundlers."}), lockFlags); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 0938cbe5b..d0b140570 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,6 +443,7 @@ struct CmdDevelop : Common, MixEnvironment auto state = getEvalState(); auto bashInstallable = std::make_shared( + this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 091af8084..b9cde5d6d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -595,7 +595,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); - auto installable = InstallableFlake( + auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), Strings{templateName == "" ? "defaultTemplate" : templateName}, Strings(attrsPathPrefixes), lockFlags); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 827f8be5a..4d275f577 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -399,7 +399,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); - InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags); + InstallableFlake installable( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + {element.source->attrPath}, + {}, + lockFlags); auto [attrPath, resolvedRef, drv] = installable.toDerivation(); From 7bd9898d5ca72ed136032590745c56826317a328 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:54:13 +0100 Subject: [PATCH 365/477] nix run: Allow program name to be set in meta.mainProgram This is useful when the program name doesn't match the package name (e.g. ripgrep vs rg). Fixes #4498. --- src/nix/app.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nix/app.cc b/src/nix/app.cc index 80acbf658..cf147c631 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -12,11 +12,16 @@ App Installable::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); + auto checkProgram = [&](const Path & program) + { + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); + }; + if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - if (!state.store->isInStore(program)) - throw Error("app program '%s' is not in the Nix store", program); + checkProgram(program); std::vector context2; for (auto & [path, name] : context) @@ -33,9 +38,17 @@ App Installable::toApp(EvalState & state) auto outPath = cursor->getAttr(state.sOutPath)->getString(); auto outputName = cursor->getAttr(state.sOutputName)->getString(); auto name = cursor->getAttr(state.sName)->getString(); + auto aMeta = cursor->maybeGetAttr("meta"); + auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; + auto mainProgram = + aMainProgram + ? aMainProgram->getString() + : DrvName(name).name; + auto program = outPath + "/bin/" + mainProgram; + checkProgram(program); return App { .context = { { drvPath, {outputName} } }, - .program = outPath + "/bin/" + DrvName(name).name, + .program = program, }; } From 1b578255245e2e1347059ad7d9171cf822c723a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:58:40 +0100 Subject: [PATCH 366/477] Document meta.mainProgram Issue #4498. --- src/nix/run.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix/run.md b/src/nix/run.md index c178e8b13..a76750376 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -43,9 +43,10 @@ program specified by the app definition. If *installable* evaluates to a derivation, it will try to execute the program `/bin/`, where *out* is the primary output store -path of the derivation and *name* is the name part of the value of the -`name` attribute of the derivation (e.g. if `name` is set to -`hello-1.10`, it will run `$out/bin/hello`). +path of the derivation and *name* is the `meta.mainProgram` attribute +of the derivation if it exists, and otherwise the name part of the +value of the `name` attribute of the derivation (e.g. if `name` is set +to `hello-1.10`, it will run `$out/bin/hello`). # Flake output attributes From cd44c0af71ace2eb8056c2b26b9249a5aa102b41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2021 19:22:37 +0100 Subject: [PATCH 367/477] Increase default stack size on Linux Workaround for #4550. --- src/nix/main.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5f4eb8918..1b68cf15b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -17,6 +17,10 @@ #include #include +#if __linux__ +#include +#endif + #include extern std::string chrootHelperName; @@ -325,6 +329,17 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + #if __linux__ + rlim_t stackSize = 64 * 1024 * 1024; + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif + return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); From 263f6dbd1cef6eb9560737f6daf963f8968a65d8 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 20:38:37 +0100 Subject: [PATCH 368/477] Don't crash nix-build when not all outputs are realised Change the `nix-build` logic for linking/printing the output paths to allow for some outputs to be missing. This might happen when the toplevel derivation didn't have to be built, either because all the required outputs were already there, or because they have all been substituted. --- src/nix-build/nix-build.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 361f9730d..d975cd16d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -518,9 +518,11 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = store->queryDerivationOutputMap(drvPath); + auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath); - auto outputPath = builtOutputs.at(outputName); + auto maybeOutputPath = builtOutputs.at(outputName); + assert(maybeOutputPath); + auto outputPath = *maybeOutputPath; if (auto store2 = store.dynamic_pointer_cast()) { std::string symlink = drvPrefix; From be1b5c4e59ca1c3504a44e2058807f7207432846 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 18:11:33 +0100 Subject: [PATCH 369/477] Test the garbage collection of CA derivations Simple test to ensure that `nix-build && nix-collect-garbage && nix-build -j0` works as it should --- tests/content-addressed.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index e8ac88609..7e32e1f28 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -48,6 +48,10 @@ testCutoff () { testGC () { nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true + clearStore + buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA + nix-collect-garbage --experimental-features ca-derivations + buildAttr rootCA 1 -j0 } testNixCommand () { From 87c8d3d702123528ac068bb703232e24431c535e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 27 Jan 2021 10:03:05 +0100 Subject: [PATCH 370/477] Register the realisations for unresolved drvs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once a build is done, get back to the original derivation, and register all the newly built outputs for this derivation. This allows Nix to work properly with derivations that don't have all their build inputs available − thus allowing garbage collection and (once it's implemented) binary substitution --- src/libstore/build/derivation-goal.cc | 54 ++++++++++++++++++++++++++- src/libstore/build/derivation-goal.hh | 3 ++ src/libstore/derivations.cc | 9 ++++- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 15 +------- src/libstore/store-api.hh | 6 --- 7 files changed, 67 insertions(+), 24 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index eeaec4f2c..315cf3f0a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -506,6 +506,7 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); + resolvedDrv = drvResolved; auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -1019,7 +1020,45 @@ void DerivationGoal::buildDone() } void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); + assert(resolvedDrv); + + // If the derivation was originally a full `Derivation` (and not just + // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` + // will be wrong otherwise + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + + auto originalHashes = staticOutputHashes(worker.store, fullDrv); + auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv->outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(originalHashes.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = worker.store.queryRealisation( + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + ); + // We've just built it, but maybe the build failed, in which case the + // realisation won't be there + if (realisation) { + auto newRealisation = *realisation; + newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + worker.store.registerDrvOutput(newRealisation); + } else { + // If we don't have a realisation, then it must mean that something + // failed when building the resolved drv + assert(!result.success()); + } + } + + // This is potentially a bit fishy in terms of error reporting. Not sure + // how to do it in a cleaner way + amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); } HookReply DerivationGoal::tryBuildHook() @@ -3804,6 +3843,19 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + auto outputHashes = staticOutputHashes(worker.store, fullDrv); + if (auto real = worker.store.queryRealisation( + DrvOutput{outputHashes.at(i.first), i.first})) { + info.known = { + .path = real->outPath, + .status = PathStatus::Valid, + }; + } + } initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 8ee0be9e1..b7b85c85d 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -48,6 +48,9 @@ struct DerivationGoal : public Goal /* The path of the derivation. */ StorePath drvPath; + /* The path of the corresponding resolved derivation */ + std::optional resolvedDrv; + /* The specific outputs that we need to build. Empty means all of them. */ StringSet wantedOutputs; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7466c7d41..4b774c42a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -756,8 +756,13 @@ std::optional Derivation::tryResolveUncached(Store & store) { StringSet newOutputNames; for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) + if (!actualPathOpt) { + warn("Input %s!%s missing, aborting the resolving", + store.printStorePath(input.first), + outputName + ); return std::nullopt; + } auto actualPath = *actualPathOpt; inputRewrites.emplace( downstreamPlaceholder(store, input.first, outputName), @@ -782,6 +787,8 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; + debug("Trying to resolve %s", store.printStorePath(drvPath)); + { auto resolutions = resolutionsCache.lock(); auto resolvedDrvIter = resolutions->find(drvPath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f45af2bac..e06c47cde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9d235ba0a..780cc0f07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 37c11fe86..2658f7617 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,7 +366,7 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); @@ -376,19 +376,6 @@ std::map> Store::queryDerivationOutputMapN return outputs; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) -{ - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto resolvedDrv = Derivation::tryResolve(*this, path); - if (resolvedDrv) { - auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); - if (isValidPath(resolvedDrvPath)) - return queryDerivationOutputMapNoResolve(resolvedDrvPath); - } - } - return queryDerivationOutputMapNoResolve(path); -} - OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9e98eb8f9..6dcd43ed1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -415,12 +415,6 @@ public: `std::nullopt`. */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* - * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve - * the derivation - */ - virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ OutputPathMap queryDerivationOutputMap(const StorePath & path); From 93d9eb78a0733c5adcbc6ee7b8a257605ae4a32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:12:24 +0100 Subject: [PATCH 371/477] Syntactic fixes Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 4b774c42a..36993ffc2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -787,7 +787,7 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; - debug("Trying to resolve %s", store.printStorePath(drvPath)); + debug("trying to resolve %s", store.printStorePath(drvPath)); { auto resolutions = resolutionsCache.lock(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e06c47cde..0962418dd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { From 0bfbd043699908bcaff1493c733ab4798b642b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:13:38 +0100 Subject: [PATCH 372/477] Don't expose the "bang" drvoutput syntax It's not fixed nor useful atm, so better keep it hidden Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 36993ffc2..7807089ca 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -757,9 +757,9 @@ std::optional Derivation::tryResolveUncached(Store & store) { for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); if (!actualPathOpt) { - warn("Input %s!%s missing, aborting the resolving", - store.printStorePath(input.first), - outputName + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(input.first) ); return std::nullopt; } From 4bc28c44f258f4f8c8a3935d1acf746f6abe3d8f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 14:41:49 +0100 Subject: [PATCH 373/477] Store the output hashes in the initialOutputs of the drv goal That way we 1. Don't have to recompute them several times 2. Can compute them in a place where we know the type of the parent derivation, meaning that we don't need the casting dance we had before --- src/libstore/build/derivation-goal.cc | 49 ++++++++++++++++----------- src/libstore/build/derivation-goal.hh | 1 + 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 315cf3f0a..d8a89a2d0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -124,6 +124,17 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation , buildMode(buildMode) { this->drv = std::make_unique(BasicDerivation(drv)); + + auto outputHashes = staticOutputHashes(worker.store, drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", @@ -258,8 +269,20 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); + auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); + + auto outputHashes = staticOutputHashes(worker.store, *fullDrv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); + drv = std::unique_ptr(fullDrv); haveDerivation(); } @@ -1022,14 +1045,6 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { assert(resolvedDrv); - // If the derivation was originally a full `Derivation` (and not just - // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` - // will be wrong otherwise - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - - auto originalHashes = staticOutputHashes(worker.store, fullDrv); auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); // `wantedOutputs` might be empty, which means “all the outputs” @@ -1038,7 +1053,7 @@ void DerivationGoal::resolvedFinished() { realWantedOutputs = resolvedDrv->outputNames(); for (auto & wantedOutput : realWantedOutputs) { - assert(originalHashes.count(wantedOutput) != 0); + assert(initialOutputs.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0); auto realisation = worker.store.queryRealisation( DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} @@ -1047,7 +1062,7 @@ void DerivationGoal::resolvedFinished() { // realisation won't be there if (realisation) { auto newRealisation = *realisation; - newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something @@ -3829,9 +3844,8 @@ void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; + InitialOutput & info = initialOutputs.at(i.first); + info.wanted = wantOutput(i.first, wantedOutputs); if (i.second) { auto outputPath = *i.second; info.known = { @@ -3844,19 +3858,14 @@ void DerivationGoal::checkPathValidity() }; } if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - auto outputHashes = staticOutputHashes(worker.store, fullDrv); if (auto real = worker.store.queryRealisation( - DrvOutput{outputHashes.at(i.first), i.first})) { + DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; } } - initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index b7b85c85d..761100d3a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -37,6 +37,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; + Hash outputHash; std::optional known; }; From f483b623e98a0feb2568e5be076b533c5838ba32 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Feb 2021 08:16:12 +0100 Subject: [PATCH 374/477] Remove the drv resolution caching mechanism It isn't needed anymore now that don't need to eagerly resolve everything like we used to do. So we can safely get rid of it --- src/libstore/derivations.cc | 34 +--------------------------------- src/libstore/derivations.hh | 4 ---- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7807089ca..6d0742b4f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolveUncached(Store & store) { +std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -776,36 +776,4 @@ std::optional Derivation::tryResolveUncached(Store & store) { return resolved; } -std::optional Derivation::tryResolve(Store& store) -{ - auto drvPath = writeDerivation(store, *this, NoRepair, false); - return Derivation::tryResolve(store, drvPath); -} - -std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - - debug("trying to resolve %s", store.printStorePath(drvPath)); - - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(drvPath); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(drvPath); - auto attempt = drv.tryResolveUncached(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); - return *attempt; -} - } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 3d8f19aef..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,14 +138,10 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); - static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - -private: - std::optional tryResolveUncached(Store & store); }; From ae4260f0a79c5cbb7c88ddbef1f512e0771f7414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 15 Feb 2021 10:20:54 +0000 Subject: [PATCH 375/477] Generate installer script for each PR/push This works by using Cachix feature of serving a file from a store path. --- .github/workflows/test.yml | 44 +++++++++++- flake.nix | 70 ++++++++++---------- scripts/prepare-installer-for-github-actions | 10 +++ 3 files changed, 89 insertions(+), 35 deletions(-) create mode 100755 scripts/prepare-installer-for-github-actions diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 021642f4c..bde6106e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,10 +8,52 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} + env: + CACHIX_NAME: nix-ci steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v12 + - uses: cachix/cachix-action@v8 + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) + installer: + if: github.event_name == 'push' + needs: tests + runs-on: ubuntu-latest + env: + CACHIX_NAME: nix-ci + outputs: + installerURL: ${{ steps.prepare-installer.outputs.installerURL }} + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v12 + - uses: cachix/cachix-action@v8 + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + - id: prepare-installer + run: scripts/prepare-installer-for-github-actions + installer_test: + if: github.event_name == 'push' + needs: installer + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + CACHIX_NAME: nix-ci + steps: + - uses: actions/checkout@v2.3.4 + - uses: cachix/install-nix-action@master + with: + install_url: '${{needs.installer.outputs.installerURL}}' + install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve' + - run: nix-instantiate -E 'builtins.currentTime' --eval + \ No newline at end of file diff --git a/flake.nix b/flake.nix index 8c60934e6..fc334ac5b 100644 --- a/flake.nix +++ b/flake.nix @@ -109,6 +109,40 @@ ]; }; + installScriptFor = systems: + with nixpkgsFor.x86_64-linux; + runCommand "installer-script" + { buildInputs = [ nix ]; + } + '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix + # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + '' + ) + systems + } --replace '@nixVersion@' ${version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -313,40 +347,8 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = - with nixpkgsFor.x86_64-linux; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix - # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (system: - '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - '' - ) - [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] - } \ - --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. coverage = diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions new file mode 100755 index 000000000..92d930384 --- /dev/null +++ b/scripts/prepare-installer-for-github-actions @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) +installerHash=$(echo $script | cut -b12-43 -) + +installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install + +echo "::set-output name=installerURL::$installerURL" From 22aec8cef43e77bba356d099868fe0a6e7545b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sun, 21 Feb 2021 14:55:45 +0000 Subject: [PATCH 376/477] fix installer script --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 0eaf25bb3..7d25f7bd7 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -60,7 +60,7 @@ case "$(uname -s).$(uname -m)" in esac # Use this command-line option to fetch the tarballs using nar-serve or Cachix -if "${1:---tarball-url-prefix}"; then +if [ "${1:-}" = "--tarball-url-prefix" ]; then if [ -z "${2:-}" ]; then oops "missing argument for --tarball-url-prefix" fi From 2de232d2b301b2f0854b9fa715ab085612c85e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 16 Feb 2021 14:32:12 +0100 Subject: [PATCH 377/477] Add x86_64 compute levels as additional system types When performing distributed builds of machine learning packages, it would be nice if builders without the required SIMD instructions can be excluded as build nodes. Since x86_64 has accumulated a large number of different instruction set extensions, listing all possible extensions would be unwieldy. AMD, Intel, Red Hat, and SUSE have recently defined four different microarchitecture levels that are now part of the x86-64 psABI supplement and will be used in glibc 2.33: https://gitlab.com/x86-psABIs/x86-64-ABI https://lwn.net/Articles/844831/ This change uses libcpuid to detect CPU features and then uses them to add the supported x86_64 levels to the additional system types. For example on a Ryzen 3700X: $ ~/aps/bin/nix -vv --version | grep "Additional system" Additional system types: i686-linux, x86_64-v1-linux, x86_64-v2-linux, x86_64-v3-linux --- Makefile.config.in | 1 + configure.ac | 8 ++++ flake.nix | 3 +- src/libstore/globals.cc | 24 +++++++---- src/libutil/compute-levels.cc | 80 +++++++++++++++++++++++++++++++++++ src/libutil/compute-levels.hh | 7 +++ src/libutil/local.mk | 4 ++ tests/compute-levels.sh | 7 +++ tests/local.mk | 3 +- 9 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 src/libutil/compute-levels.cc create mode 100644 src/libutil/compute-levels.hh create mode 100644 tests/compute-levels.sh diff --git a/Makefile.config.in b/Makefile.config.in index d1e59e4e7..9d0500e48 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,6 +9,7 @@ CXXFLAGS = @CXXFLAGS@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ +HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ diff --git a/configure.ac b/configure.ac index 2047ed8d2..a24287ff6 100644 --- a/configure.ac +++ b/configure.ac @@ -218,6 +218,14 @@ LDFLAGS="-lz $LDFLAGS" # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) +# Look for libcpuid. +if test "$machine_name" = "x86_64"; then + PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"]) + have_libcpuid=1 + AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid]) +fi +AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) + # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then diff --git a/flake.nix b/flake.nix index 8c60934e6..3ad7cca97 100644 --- a/flake.nix +++ b/flake.nix @@ -91,7 +91,8 @@ gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isx86_64 libcpuid; awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0531aad9f..df07aee9b 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -3,6 +3,7 @@ #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "compute-levels.hh" #include #include @@ -133,24 +134,29 @@ StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultExtraPlatforms() { + StringSet extraPlatforms; + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) - return StringSet{"i686-linux"}; -#if __APPLE__ + extraPlatforms.insert("i686-linux"); + +#if __linux__ + StringSet levels = computeLevels(); + for (auto iter = levels.begin(); iter != levels.end(); ++iter) + extraPlatforms.insert(*iter + "-linux"); +#elif __APPLE__ // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") - return StringSet{"aarch64-darwin"}; + extraPlatforms.insert("aarch64-darwin"); else if (std::string{SYSTEM} == "aarch64-darwin") - return StringSet{"x86_64-darwin"}; - else - return StringSet{}; + extraPlatforms.insert("x86_64-darwin"); } #endif - else - return StringSet{}; + + return extraPlatforms; } bool Settings::isExperimentalFeatureEnabled(const std::string & name) diff --git a/src/libutil/compute-levels.cc b/src/libutil/compute-levels.cc new file mode 100644 index 000000000..19eaedfa8 --- /dev/null +++ b/src/libutil/compute-levels.cc @@ -0,0 +1,80 @@ +#include "types.hh" + +#if HAVE_LIBCPUID +#include +#endif + +namespace nix { + +#if HAVE_LIBCPUID + +StringSet computeLevels() { + StringSet levels; + + if (!cpuid_present()) + return levels; + + cpu_raw_data_t raw; + cpu_id_t data; + + if (cpuid_get_raw_data(&raw) < 0) + return levels; + + if (cpu_identify(&raw, &data) < 0) + return levels; + + if (!(data.flags[CPU_FEATURE_CMOV] && + data.flags[CPU_FEATURE_CX8] && + data.flags[CPU_FEATURE_FPU] && + data.flags[CPU_FEATURE_FXSR] && + data.flags[CPU_FEATURE_MMX] && + data.flags[CPU_FEATURE_SSE] && + data.flags[CPU_FEATURE_SSE2])) + return levels; + + levels.insert("x86_64-v1"); + + if (!(data.flags[CPU_FEATURE_CX16] && + data.flags[CPU_FEATURE_LAHF_LM] && + data.flags[CPU_FEATURE_POPCNT] && + // SSE3 + data.flags[CPU_FEATURE_PNI] && + data.flags[CPU_FEATURE_SSSE3] && + data.flags[CPU_FEATURE_SSE4_1] && + data.flags[CPU_FEATURE_SSE4_2])) + return levels; + + levels.insert("x86_64-v2"); + + if (!(data.flags[CPU_FEATURE_AVX] && + data.flags[CPU_FEATURE_AVX2] && + data.flags[CPU_FEATURE_F16C] && + data.flags[CPU_FEATURE_FMA3] && + // LZCNT + data.flags[CPU_FEATURE_ABM] && + data.flags[CPU_FEATURE_MOVBE])) + return levels; + + levels.insert("x86_64-v3"); + + if (!(data.flags[CPU_FEATURE_AVX512F] && + data.flags[CPU_FEATURE_AVX512BW] && + data.flags[CPU_FEATURE_AVX512CD] && + data.flags[CPU_FEATURE_AVX512DQ] && + data.flags[CPU_FEATURE_AVX512VL])) + return levels; + + levels.insert("x86_64-v4"); + + return levels; +} + +#else + +StringSet computeLevels() { + return StringSet{}; +} + +#endif // HAVE_LIBCPUID + +} diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh new file mode 100644 index 000000000..8ded295f9 --- /dev/null +++ b/src/libutil/compute-levels.hh @@ -0,0 +1,7 @@ +#include "types.hh" + +namespace nix { + +StringSet computeLevels(); + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index ae7eb67ad..5341c58e6 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context + +ifeq ($(HAVE_LIBCPUID), 1) + libutil_LDFLAGS += -lcpuid +endif diff --git a/tests/compute-levels.sh b/tests/compute-levels.sh new file mode 100644 index 000000000..e4322dfa1 --- /dev/null +++ b/tests/compute-levels.sh @@ -0,0 +1,7 @@ +source common.sh + +if [[ $(uname -ms) = "Linux x86_64" ]]; then + # x86_64 CPUs must always support the baseline + # microarchitecture level. + nix -vv --version | grep -q "x86_64-v1-linux" +fi diff --git a/tests/local.mk b/tests/local.mk index aa8b4f9bf..06be8cec1 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,7 +38,8 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ content-addressed.sh \ - build.sh + build.sh \ + compute-levels.sh # parallel.sh # build-remote-content-addressed-fixed.sh \ From 574eb2be81cc599162722659dcb95f19173c98d1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2021 15:24:14 +0100 Subject: [PATCH 378/477] Tweak error message --- src/libexpr/eval.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7271776eb..e2f2308aa 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1381,10 +1381,10 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -nix attempted to evaluate a function as a top level expression; in this case it must have its -arguments supplied either by default values, or passed explicitly with --arg or --argstr. - -https://nixos.org/manual/nix/stable/#ss-functions)", i.name); +Nix attempted to evaluate a function as a top level expression; in +this case it must have its arguments supplied either by default +values, or passed explicitly with '--arg' or '--argstr'. See +https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); } } From e2f3b2eb42a0ceca36ce00973bd2d49b1a3e6a2c Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 22 Feb 2021 16:13:09 +0100 Subject: [PATCH 379/477] Make missing auto-call arguments throw an eval error The PR #4240 changed messag of the error that was thrown when an auto-called function was missing an argument. However this change also changed the type of the error, from `EvalError` to a new `MissingArgumentError`. This broke hydra which was relying on an `EvalError` being thrown. Make `MissingArgumentError` a subclass of `EvalError` to un-break hydra. --- src/libexpr/nixexpr.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index cbe9a45bf..8df8055b3 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,7 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); -MakeError(MissingArgumentError, Error); +MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); From 35205e2e922952fc0654260a07fc3191c5afc2cc Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 22 Feb 2021 17:10:55 -0500 Subject: [PATCH 380/477] Warn about instability of plugin API --- src/libstore/globals.hh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1d968ef3e..1254698ca 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -831,6 +831,9 @@ public: command, and RegisterSetting to add new nix config settings. See the constructors for those types for more details. + Warning! These APIs are inherently unstable and may change from + release to release. + Since these files are loaded into the same address space as Nix itself, they must be DSOs compatible with the instance of Nix running at the time (i.e. compiled against the same headers, not From 6fbf3fe636bc1d9a9aba4bacb2a70191c1d6b1a7 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:48:41 +0100 Subject: [PATCH 381/477] Make the build-hook work with ca derivations - Pass it the name of the outputs rather than their output paths (as these don't exist for ca derivations) - Get the built output paths from the remote builder - Register the new received realisations --- src/build-remote/build-remote.cc | 36 +++++++++++++++++++++------ src/libstore/build/derivation-goal.cc | 9 ++++--- src/libstore/realisation.hh | 2 ++ src/libstore/remote-store.cc | 16 ++++++++++++ src/libstore/store-api.hh | 2 ++ src/libstore/worker-protocol.hh | 2 ++ 6 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5b8ab3387..c2319a3d1 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -248,7 +248,7 @@ connected: std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); - auto outputs = readStrings(source); + auto wantedOutputs = readStrings(source); AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); @@ -273,6 +273,7 @@ connected: uploadLock = -1; auto drv = store->readDerivation(*drvPath); + auto outputHashes = staticOutputHashes(*store, drv); drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); @@ -280,16 +281,35 @@ connected: if (!result.success()) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); - StorePathSet missing; - for (auto & path : outputs) - if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); + std::set missingRealisations; + StorePathSet missingPaths; + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & outputName : wantedOutputs) { + auto thisOutputHash = outputHashes.at(outputName); + auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; + if (!store->queryRealisation(thisOutputId)) { + notice("Missing output %s", outputName); + assert(result.builtOutputs.count(thisOutputId)); + auto newRealisation = result.builtOutputs.at(thisOutputId); + missingRealisations.insert(newRealisation); + missingPaths.insert(newRealisation.outPath); + } + } + } else { + auto outputPaths = drv.outputsAndOptPaths(*store); + for (auto & [outputName, hopefullyOutputPath] : outputPaths) { + assert(hopefullyOutputPath.second); + if (!store->isValidPath(*hopefullyOutputPath.second)) + missingPaths.insert(*hopefullyOutputPath.second); + } + } - if (!missing.empty()) { + if (!missingPaths.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) - for (auto & i : missing) - localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ - copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); + for (auto & path : missingPaths) + localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ + copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } return 0; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8a89a2d0..b074410b0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1159,13 +1159,14 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { + StringSet missingOutputs; + for (auto & [outputName, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); + missingOutputs.insert(outputName); + /* missingPaths.insert(status.known->path); */ } - worker_proto::write(worker.store, hook->sink, missingPaths); + worker_proto::write(worker.store, hook->sink, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 7c91d802a..fc92d3c17 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -33,6 +33,8 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +typedef std::map DrvOutputs; + struct OpaquePath { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be07f02dc..52d633372 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -12,6 +12,7 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include namespace nix { @@ -49,6 +50,21 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } +Realisation read(const Store & store, Source & from, Phantom _) +{ + std::string rawInput = readString(from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} +void write(const Store & store, Sink & out, const Realisation & realisation) +{ out << realisation.toJSON().dump(); } + +DrvOutput read(const Store & store, Source & from, Phantom _) +{ return DrvOutput::parse(readString(from)); } +void write(const Store & store, Sink & out, const DrvOutput & drvOutput) +{ out << drvOutput.to_string(); } std::optional read(const Store & store, Source & from, Phantom> _) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6dcd43ed1..ea6389ba4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -162,6 +162,8 @@ struct BuildResult non-determinism.) */ bool isNonDeterministic = false; + DrvOutputs builtOutputs; + /* The start/stop times of the build (or one of the rounds, if it was repeated). */ time_t startTime = 0, stopTime = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f2cdc7ca3..5e094c378 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,6 +86,8 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, Realisation); +MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(template, std::set); From 5687564a27bee692f68a78b897a2d68715f6a3ce Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:50:44 +0100 Subject: [PATCH 382/477] LocalStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/build/entry-points.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 9f97d40ba..99b3fa070 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -58,6 +58,26 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat result.status = BuildResult::MiscFailure; result.errorMsg = e.msg(); } + // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's + // extended to return the full realisation for each output + auto staticDrvOutputs = drv.outputsAndOptPaths(*this); + auto outputHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, staticOutput] : staticDrvOutputs) { + auto outputId = DrvOutput{outputHashes.at(outputName), outputName}; + if (staticOutput.second) + result.builtOutputs.insert_or_assign( + outputId, + Realisation{ outputId, *staticOutput.second} + ); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto realisation = this->queryRealisation(outputId); + if (realisation) + result.builtOutputs.insert_or_assign( + outputId, + *realisation + ); + } + } return result; } From a2b69660a9b326b95d48bd222993c5225bbd5b5f Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:50:44 +0100 Subject: [PATCH 383/477] LegacySSHStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/legacy-ssh-store.cc | 4 +++- src/libstore/serve-protocol.hh | 2 +- src/nix-store/nix-store.cc | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 253c0033e..daf78042f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -258,7 +258,9 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { + status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + } return status; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 9fae6d534..0a17387cb 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x205 +#define SERVE_PROTOCOL_VERSION 0x206 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 37191b9e6..559fd5355 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -905,6 +905,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; + if (GET_PROTOCOL_MINOR(clientVersion >= 5)) { + worker_proto::write(*store, out, status.builtOutputs); + } + break; } From 27b5747ca7b5599768083dde5fa4d36bfbb0f66f Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 25 Jan 2021 11:08:38 +0100 Subject: [PATCH 384/477] RemoteStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/daemon.cc | 3 +++ src/libstore/remote-store.cc | 4 ++++ src/libstore/worker-protocol.hh | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ba5788b64..ba7959263 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -575,6 +575,9 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; + if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) { + worker_proto::write(*store, to, res.builtOutputs); + } break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 52d633372..0d884389a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -680,6 +680,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD unsigned int status; conn->from >> status >> res.errorMsg; res.status = (BuildResult::Status) status; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) { + auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + res.builtOutputs = builtOutputs; + } return res; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5e094c378..95f08bc9a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11b +#define PROTOCOL_VERSION 0x11c #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From 8c385d16eeeb26a912d213c5689d9f9a78020bc7 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 09:35:10 +0100 Subject: [PATCH 385/477] Also send ca outputs to the build hook Otherwise they don't get registered, triggering an assertion failure at some point later --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b074410b0..096f24029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,8 +1161,8 @@ HookReply DerivationGoal::tryBuildHook() { StringSet missingOutputs; for (auto & [outputName, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; + // XXX: Does this include known CA outputs? + if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); /* missingPaths.insert(status.known->path); */ } From 69666b951ee06733ed420cb4cd408a19e42c6e43 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 09:36:24 +0100 Subject: [PATCH 386/477] build-remote: Always register the missing outputs It's possible that all the paths are already there, but just not associated to the current drv output --- src/build-remote/build-remote.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c2319a3d1..228aba35a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,6 +311,13 @@ connected: localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } + // XXX: Should e done as part of `copyPaths` + for (auto & realisation : missingRealisations) { + // Should hold, because if the feature isn't enabled the set + // of missing realisations should be empty + settings.requireExperimentalFeature("ca-derivations"); + store->registerDrvOutput(realisation); + } return 0; } From 527da736905730e70725bf4b3556d61267d220ba Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:02:03 +0100 Subject: [PATCH 387/477] Properly bypass the registering step when all outputs are present There was already some logic for that, but it didn't handle the case of content-addressed outputs, so extend it a bit for that --- src/libstore/build/derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 096f24029..6052b625d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3001,11 +3001,11 @@ void DerivationGoal::registerOutputs() */ if (hook) { bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) + for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!outputPath || !worker.store.isValidPath(*outputPath)) allValid = false; else - finalOutputs.insert_or_assign(i.first, *i.second.second); + finalOutputs.insert_or_assign(outputName, *outputPath); } if (allValid) return; } From c32168c9bc161e0c9cea027853895971699510cb Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:28:00 +0100 Subject: [PATCH 388/477] Test the remote building of ca derivations --- tests/build-hook-ca.nix | 16 ++++++++++++---- tests/build-remote-content-addressed-fixed.sh | 5 ----- tests/build-remote-content-addressed-floating.sh | 7 +++++++ tests/local.mk | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) delete mode 100644 tests/build-remote-content-addressed-fixed.sh create mode 100644 tests/build-remote-content-addressed-floating.sh diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca.nix index 98db473fc..67295985f 100644 --- a/tests/build-hook-ca.nix +++ b/tests/build-hook-ca.nix @@ -11,6 +11,7 @@ let args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; + __contentAddressed = true; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; @@ -19,7 +20,6 @@ let name = "build-remote-input-1"; buildCommand = "echo FOO > $out"; requiredSystemFeatures = ["foo"]; - outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; }; input2 = mkDerivation { @@ -27,7 +27,16 @@ let name = "build-remote-input-2"; buildCommand = "echo BAR > $out"; requiredSystemFeatures = ["bar"]; - outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + read x < ${input2} + echo $x BAZ > $out + ''; + requiredSystemFeatures = ["baz"]; }; in @@ -38,8 +47,7 @@ in buildCommand = '' read x < ${input1} - read y < ${input2} + read y < ${input3} echo "$x $y" > $out ''; - outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0="; } diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh deleted file mode 100644 index 1408a19d5..000000000 --- a/tests/build-remote-content-addressed-fixed.sh +++ /dev/null @@ -1,5 +0,0 @@ -source common.sh - -file=build-hook-ca.nix - -source build-remote.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh new file mode 100644 index 000000000..cbb75729b --- /dev/null +++ b/tests/build-remote-content-addressed-floating.sh @@ -0,0 +1,7 @@ +source common.sh + +file=build-hook-ca.nix + +sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf + +source build-remote.sh diff --git a/tests/local.mk b/tests/local.mk index aa8b4f9bf..9bde2322f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + build-remote-content-addressed-floating.sh \ ssh-relay.sh \ nar-access.sh \ structured-attrs.sh \ @@ -40,7 +41,6 @@ nix_tests = \ content-addressed.sh \ build.sh # parallel.sh - # build-remote-content-addressed-fixed.sh \ install-tests += $(foreach x, $(nix_tests), tests/$(x)) From ba1a256d0875592b28d902f3e40663b2adedfe9c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 23 Feb 2021 14:12:11 +0100 Subject: [PATCH 389/477] Make `DerivationGoal::drv` a full Derivation This field used to be a `BasicDerivation`, but this `BasicDerivation` was downcasted to a `Derivation` when needed (implicitely or not), so we might as well make it a full `Derivation` and upcast it when needed. This also allows getting rid of a weird duplication in the way we compute the static output hashes for the derivation. We had to do it differently and in a different place depending on whether the derivation was a full derivation or just a basic drv, but we can now do it unconditionally on the full derivation. Fix #4559 --- src/libstore/build/derivation-goal.cc | 37 ++++++++++----------------- src/libstore/build/derivation-goal.hh | 2 +- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8a89a2d0..804a79e4c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -123,17 +123,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - this->drv = std::make_unique(BasicDerivation(drv)); - - auto outputHashes = staticOutputHashes(worker.store, drv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); + this->drv = std::make_unique(drv); state = &DerivationGoal::haveDerivation; name = fmt( @@ -271,18 +261,8 @@ void DerivationGoal::loadDerivation() auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); - auto outputHashes = staticOutputHashes(worker.store, *fullDrv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); - /* Get the derivation. */ - drv = std::unique_ptr(fullDrv); + drv = std::unique_ptr(fullDrv); haveDerivation(); } @@ -301,6 +281,16 @@ void DerivationGoal::haveDerivation() if (i.second.second) worker.store.addTempRoot(*i.second.second); + auto outputHashes = staticOutputHashes(worker.store, *drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + /* Check what outputs paths are not already valid. */ checkPathValidity(); bool allValid = true; @@ -3517,10 +3507,9 @@ void DerivationGoal::registerOutputs() but it's fine to do in all cases. */ if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto outputHashes = staticOutputHashes(worker.store, *drv); for (auto& [outputName, newInfo] : infos) worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{outputHashes.at(outputName), outputName}, + .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, .outPath = newInfo.path}); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 761100d3a..6dc164922 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -64,7 +64,7 @@ struct DerivationGoal : public Goal bool retrySubstitution; /* The derivation stored at drvPath. */ - std::unique_ptr drv; + std::unique_ptr drv; std::unique_ptr parsedDrv; From ec3497c1d63f4c0547d0402d92015f846f56aac7 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 07:37:04 -0500 Subject: [PATCH 390/477] Bail if plugin-files is set after plugins have been loaded. We know the flag will be ignored but the user wants it to take effect. --- src/libstore/globals.cc | 11 +++++++++++ src/libstore/globals.hh | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index df07aee9b..03294b7fe 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -243,6 +243,14 @@ void MaxBuildJobsSetting::set(const std::string & str, bool append) } +void PluginFilesSetting::set(const std::string & str, bool append) +{ + if (pluginsLoaded) + throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + BaseSetting::set(str, append); +} + + void initPlugins() { for (const auto & pluginFile : settings.pluginFiles.get()) { @@ -270,6 +278,9 @@ void initPlugins() unknown settings. */ globalConfig.reapplyUnknownSettings(); globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + settings.pluginFiles.pluginsLoaded = true; } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1254698ca..df61d6417 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -28,6 +28,23 @@ struct MaxBuildJobsSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting(Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str, bool append = false) override; +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -819,7 +836,7 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting pluginFiles{ + PluginFilesSetting pluginFiles{ this, {}, "plugin-files", R"( A list of plugin files to be loaded by Nix. Each of these files will From 98d1b64400cc7b75216fc885859883c707c18bef Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 09:37:43 -0500 Subject: [PATCH 391/477] Initialize plugins after handling initial command line flags This is technically a breaking change, since attempting to set plugin files after the first non-flag argument will now throw an error. This is acceptable given the relative lack of stability in a plugin interface and the need to tie the knot somewhere once plugins can actually define new subcommands. --- doc/manual/src/release-notes/rl-2.4.md | 7 +++++++ src/build-remote/build-remote.cc | 3 +++ src/libmain/common-args.cc | 7 +++++++ src/libmain/common-args.hh | 6 +++++- src/libstore/globals.cc | 1 + src/libutil/args.cc | 8 ++++++++ src/libutil/args.hh | 4 ++++ src/nix-build/nix-build.cc | 2 -- src/nix-channel/nix-channel.cc | 2 -- src/nix-collect-garbage/nix-collect-garbage.cc | 2 -- src/nix-copy-closure/nix-copy-closure.cc | 2 -- src/nix-env/nix-env.cc | 2 -- src/nix-instantiate/nix-instantiate.cc | 2 -- src/nix-store/nix-store.cc | 2 -- src/nix/daemon.cc | 2 -- src/nix/main.cc | 2 -- src/nix/prefetch.cc | 2 -- tests/plugins.sh | 2 +- 18 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.4.md diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md new file mode 100644 index 000000000..26ba70904 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -0,0 +1,7 @@ +# Release 2.4 (202X-XX-XX) + + - It is now an error to modify the `plugin-files` setting via a + command-line flag that appears after the first non-flag argument + to any command, including a subcommand to `nix`. For example, + `nix-instantiate default.nix --plugin-files ""` must now become + `nix-instantiate --plugin-files "" default.nix`. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5b8ab3387..f784b5160 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv) unsetenv("DISPLAY"); unsetenv("SSH_ASKPASS"); + /* If we ever use the common args framework, make sure to + remove initPlugins below and initialize settings first. + */ if (argc != 2) throw UsageError("called without required arguments"); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index ff96ee7d5..c43e9ebd2 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -79,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName) hiddenCategories.insert(cat); } +void MixCommonArgs::initialFlagsProcessed() +{ + initPlugins(); + pluginsInited(); +} + + } diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 8e53a7361..31bdf527a 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -7,10 +7,14 @@ namespace nix { //static constexpr auto commonArgsCategory = "Miscellaneous common options"; static constexpr auto loggingCategory = "Logging-related options"; -struct MixCommonArgs : virtual Args +class MixCommonArgs : public virtual Args { + void initialFlagsProcessed() override; +public: string programName; MixCommonArgs(const string & programName); +protected: + virtual void pluginsInited() {} }; struct MixDryRun : virtual Args diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 03294b7fe..2780e0bf5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -253,6 +253,7 @@ void PluginFilesSetting::set(const std::string & str, bool append) void initPlugins() { + assert(!settings.pluginFiles.pluginsLoaded); for (const auto & pluginFile : settings.pluginFiles.get()) { Paths pluginFiles; try { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 9377fe4c0..eb11fd64b 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -60,6 +60,7 @@ void Args::parseCmdline(const Strings & _cmdline) verbosity = lvlError; } + bool argsSeen = false; for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -88,6 +89,10 @@ void Args::parseCmdline(const Strings & _cmdline) throw UsageError("unrecognised flag '%1%'", arg); } else { + if (!argsSeen) { + argsSeen = true; + initialFlagsProcessed(); + } pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) @@ -96,6 +101,9 @@ void Args::parseCmdline(const Strings & _cmdline) } processArgs(pendingArgs, true); + + if (!argsSeen) + initialFlagsProcessed(); } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 88f068087..4721c21df 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -132,6 +132,10 @@ protected: std::set hiddenCategories; + /* Called after all command line flags before the first non-flag + argument (if any) have been processed. */ + virtual void initialFlagsProcessed() {} + public: void addFlag(Flag && flag); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d975cd16d..7b4a53919 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -240,8 +240,6 @@ static void main_nix_build(int argc, char * * argv) myArgs.parseCmdline(args); - initPlugins(); - if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 57189d557..3272c6125 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -196,8 +196,6 @@ static int main_nix_channel(int argc, char ** argv) return true; }); - initPlugins(); - switch (cmd) { case cNone: throw UsageError("no command specified"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index c1769790a..4f953fab4 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -74,8 +74,6 @@ static int main_nix_collect_garbage(int argc, char * * argv) return true; }); - initPlugins(); - auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ad2e06067..5e8cc515b 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -43,8 +43,6 @@ static int main_nix_copy_closure(int argc, char ** argv) return true; }); - initPlugins(); - if (sshHost.empty()) throw UsageError("no host name specified"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106a78fc4..0f10a4cbb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1420,8 +1420,6 @@ static int main_nix_env(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index ea2e85eb0..95903d882 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -149,8 +149,6 @@ static int main_nix_instantiate(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 37191b9e6..e17b38c3c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1067,8 +1067,6 @@ static int main_nix_store(int argc, char * * argv) return true; }); - initPlugins(); - if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 26006167d..2cf2a04c9 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -326,8 +326,6 @@ static int main_nix_daemon(int argc, char * * argv) return true; }); - initPlugins(); - runDaemon(stdio); return 0; diff --git a/src/nix/main.cc b/src/nix/main.cc index 1b68cf15b..b078366fa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -283,8 +283,6 @@ void mainWrapped(int argc, char * * argv) if (completions) return; - initPlugins(); - if (args.showVersion) { printVersion(programName); return; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index a831dcd15..b7da3ea5a 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -171,8 +171,6 @@ static int main_nix_prefetch_url(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (args.size() > 2) throw UsageError("too many arguments"); diff --git a/tests/plugins.sh b/tests/plugins.sh index 50bfaf7e9..e22bf4408 100644 --- a/tests/plugins.sh +++ b/tests/plugins.sh @@ -2,6 +2,6 @@ source common.sh set -o pipefail -res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) +res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] From f6c5b05488c588964f51ce97ad2c297fbca7ce96 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 10:04:47 -0500 Subject: [PATCH 392/477] Respect command registrations in plugins. --- doc/manual/src/release-notes/rl-2.4.md | 1 + src/libutil/args.cc | 4 ++-- src/nix/main.cc | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 26ba70904..f7ab9f6ad 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -5,3 +5,4 @@ to any command, including a subcommand to `nix`. For example, `nix-instantiate default.nix --plugin-files ""` must now become `nix-instantiate --plugin-files "" default.nix`. + - Plugins that add new `nix` subcommands are now actually respected. diff --git a/src/libutil/args.cc b/src/libutil/args.cc index eb11fd64b..75eb19d28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -306,8 +306,8 @@ Strings argvToStrings(int argc, char * * argv) return args; } -MultiCommand::MultiCommand(const Commands & commands) - : commands(commands) +MultiCommand::MultiCommand(const Commands & commands_) + : commands(commands_) { expectArgs({ .label = "subcommand", diff --git a/src/nix/main.cc b/src/nix/main.cc index b078366fa..06e221682 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -159,6 +159,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs #include "nix.md" ; } + + // Plugins may add new subcommands. + void pluginsInited() override + { + commands = RegisterCommand::getCommandsFor({}); + } }; static void showHelp(std::vector subcommand) From 1130b2882415b003f5ba2fc0b5466b573fe1b05a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 24 Feb 2021 20:52:22 -0500 Subject: [PATCH 393/477] distributed builds: load remote builder host key from the machines file This is already used by Hydra, and is very useful when materializing a remote builder list from service discovery. This allows the service discovery tool to only sync one file instead of two. --- .../src/advanced-topics/distributed-builds.md | 10 +++++++--- src/libstore/legacy-ssh-store.cc | 2 ++ src/libstore/machines.cc | 6 ++++++ src/libstore/ssh-store.cc | 2 ++ src/libstore/ssh.cc | 16 ++++++++++++++-- src/libstore/ssh.hh | 3 ++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index c6966a50b..580b36736 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -37,7 +37,7 @@ then you need to ensure that the `PATH` of non-interactive login shells contains Nix. > **Warning** -> +> > If you are building via the Nix daemon, it is the Nix daemon user > account (that is, `root`) that should have SSH access to the remote > machine. If you can’t or don’t want to configure `root` to be able to @@ -52,7 +52,7 @@ example, the following command allows you to build a derivation for ```console $ uname Linux - + $ nix build \ '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ --builders 'ssh://mac x86_64-darwin' @@ -103,7 +103,7 @@ default, set it to `-`. ```nix requiredSystemFeatures = [ "kvm" ]; ``` - + will cause the build to be performed on a machine that has the `kvm` feature. @@ -112,6 +112,10 @@ default, set it to `-`. features appear in the derivation’s `requiredSystemFeatures` attribute.. +8. The (base64-encoded) public host key of the remote machine. If omitted, SSH + will use its regular known-hosts file. Specifically, the field is calculated + via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. + For example, the machine specification nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 253c0033e..99b0bb5a8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -15,6 +15,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig using StoreConfig::StoreConfig; const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; @@ -59,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor , master( host, sshKey, + sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, compress, diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 7db2556f4..b42e5e434 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -54,9 +54,15 @@ ref Machine::openStore() const { if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; storeParams["log-fd"] = "4"; + } + + if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { if (sshKey != "") storeParams["ssh-key"] = sshKey; + if (sshPublicHostKey != "") + storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; } + { auto & fs = storeParams["system-features"]; auto append = [&](auto feats) { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 17c258201..f2caf2aeb 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig using RemoteStoreConfig::RemoteStoreConfig; const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; @@ -34,6 +35,7 @@ public: , master( host, sshKey, + sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, compress) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 84548a6e4..235eed37a 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,24 +2,37 @@ namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) +SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) : host(host) , fakeSSH(host == "localhost") , keyFile(keyFile) + , sshPublicHostKey(sshPublicHostKey) , useMaster(useMaster && !fakeSSH) , compress(compress) , logFD(logFD) { if (host == "" || hasPrefix(host, "-")) throw Error("invalid SSH host name '%s'", host); + + auto state(state_.lock()); + state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); } void SSHMaster::addCommonSSHOpts(Strings & args) { + auto state(state_.lock()); + for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) args.push_back(i); if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); + if (!sshPublicHostKey.empty()) { + Path fileName = (Path) *state->tmpDir + "/host-key"; + auto p = host.rfind("@"); + string thost = p != string::npos ? string(host, p + 1) : host; + writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n"); + args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName}); + } if (compress) args.push_back("-C"); } @@ -87,7 +100,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 4f0f0bd29..dabbcedda 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -12,6 +12,7 @@ private: const std::string host; bool fakeSSH; const std::string keyFile; + const std::string sshPublicHostKey; const bool useMaster; const bool compress; const int logFD; @@ -29,7 +30,7 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); + SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); struct Connection { From 2e199673a523fa81de31ffdd2a25976ce0814631 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 19:43:53 +0100 Subject: [PATCH 394/477] Use `RealisedPath`s in `copyPaths` That way we can copy the realisations too (in addition to the store paths themselves) --- src/libstore/store-api.cc | 31 ++++++++++++++---------- src/libstore/store-api.hh | 9 +++---- src/nix-copy-closure/nix-copy-closure.cc | 6 ++--- src/nix/copy.cc | 13 +++++----- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2658f7617..529c34de5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -783,6 +783,24 @@ void copyStorePath(ref srcStore, ref dstStore, } +std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + StorePathSet storePaths; + std::set realisations; + for (auto path : paths) { + storePaths.insert(path.path()); + if (auto realisation = std::get_if(&path.raw)) + realisations.insert(*realisation); + } + auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + + return pathsMap; +} + std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { @@ -796,7 +814,6 @@ std::map copyPaths(ref srcStore, ref dstStor for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -871,21 +888,9 @@ std::map copyPaths(ref srcStore, ref dstStor nrDone++; showProgress(); }); - return pathsMap; } - -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, - SubstituteFlag substitute) -{ - StorePathSet closure; - srcStore->computeFSClosure(storePaths, closure); - copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); -} - - std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) { std::string path; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6dcd43ed1..63b26422a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,15 +752,12 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const StorePathSet & storePaths, + const RealisedPath::Set&, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); - - -/* Copy the closure of the specified paths from one store to another. */ -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet& paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 5e8cc515b..02ccbe541 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv) auto to = toMode ? openStore(remoteUri) : openStore(); auto from = toMode ? openStore() : openStore(remoteUri); - StorePathSet storePaths2; + RealisedPath::Set storePaths2; for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - StorePathSet closure; - from->computeFSClosure(storePaths2, closure, false, includeOutputs); + RealisedPath::Set closure; + RealisedPath::closure(*from, storePaths2, closure); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c56a1def1..f59f7c76b 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : StorePathsCommand +struct CmdCopy : RealisedPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; - using StorePathsCommand::run; + using RealisedPathsCommand::run; CmdCopy() - : StorePathsCommand(true) + : RealisedPathsCommand(true) { addFlag({ .longName = "from", @@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - StorePathsCommand::run(store); + RealisedPathsCommand::run(store); } - void run(ref srcStore, StorePaths storePaths) override + void run(ref srcStore, std::vector paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), + copyPaths( + srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), NoRepair, checkSigs, substitute); } }; From aead35531a0630b19e41348e103b2d105e2d8dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:37:05 +0100 Subject: [PATCH 395/477] Add a test for the copy of CA paths --- tests/local.mk | 1 + tests/nix-copy-content-addressed.sh | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 tests/nix-copy-content-addressed.sh diff --git a/tests/local.mk b/tests/local.mk index 06be8cec1..a504e397e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,6 +38,7 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ content-addressed.sh \ + nix-copy-content-addressed.sh \ build.sh \ compute-levels.sh # parallel.sh diff --git a/tests/nix-copy-content-addressed.sh b/tests/nix-copy-content-addressed.sh new file mode 100755 index 000000000..2e0dea2d2 --- /dev/null +++ b/tests/nix-copy-content-addressed.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +source common.sh + +# Globally enable the ca derivations experimental flag +sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +ensureCorrectlyCopied () { + attrPath="$1" + nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath" +} + +testOneCopy () { + clearStore + rm -rf "$REMOTE_STORE_DIR" + + attrPath="$1" + nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + + ensureCorrectlyCopied "$attrPath" + + # Ensure that we can copy back what we put in the store + clearStore + nix copy --from $REMOTE_STORE \ + --file ./content-addressed.nix "$attrPath" \ + --no-check-sigs +} + +for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do + testOneCopy "$attrPath" +done From f67ff1f5756018387a2d23c8f6772580192d30ad Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 17:58:28 +0100 Subject: [PATCH 396/477] Don't crash when copying realisations to a non-ca remote Rather throw a proper exception, and catch&log it on the client side --- src/libstore/globals.cc | 7 ++++++- src/libstore/globals.hh | 18 ++++++++++++++---- src/libstore/local-store.cc | 1 + src/libstore/store-api.cc | 14 ++++++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2780e0bf5..8d44003f4 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) return std::find(f.begin(), f.end(), name) != f.end(); } +MissingExperimentalFeature::MissingExperimentalFeature(std::string feature) + : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature) + , missingFeature(feature) + {} + void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + throw MissingExperimentalFeature(name); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index df61d6417..25351f55c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,6 +45,16 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +/* MakeError(MissingExperimentalFeature, Error); */ +class MissingExperimentalFeature: public Error +{ +public: + std::string missingFeature; + + MissingExperimentalFeature(std::string feature); + virtual const char* sname() const override { return "MissingExperimentalFeature"; } +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -632,7 +642,7 @@ public: is `root`. > **Warning** - > + > > Adding a user to `trusted-users` is essentially equivalent to > giving that user root access to the system. For example, the user > can set `sandbox-paths` and thereby obtain read access to @@ -722,13 +732,13 @@ public: The program executes with no arguments. The program's environment contains the following environment variables: - - `DRV_PATH` + - `DRV_PATH` The derivation for the built paths. Example: `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - `OUT_PATHS` + - `OUT_PATHS` Output paths of the built derivation, separated by a space character. @@ -759,7 +769,7 @@ public: documentation](https://ec.haxx.se/usingcurl-netrc.html). > **Note** - > + > > This must be an absolute path, and `~` is not resolved. For > example, `~/.netrc` won't resolve to your home directory's > `.netrc`. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0962418dd..90fb4a4bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { + settings.requireExperimentalFeature("ca-derivations"); auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 529c34de5..ac1d8ee2c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -794,8 +794,18 @@ std::map copyPaths(ref srcStore, ref dstStor realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); - for (auto& realisation : realisations) { - dstStore->registerDrvOutput(realisation); + try { + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + } catch (MissingExperimentalFeature & e) { + // Don't fail if the remote doesn't support CA derivations is it might + // not be whithin our control to change that, and we might still want + // to at least copy the output paths. + if (e.missingFeature == "ca-derivations") + ignoreException(); + else + throw; } return pathsMap; From 3b76f8f252c12fbeb49aa2f6f695b4622e9fcc5d Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 18:02:26 +0100 Subject: [PATCH 397/477] Ensure that the ca-derivations bit is set when copying realisations This should already hold, but better ensure it for future-proof-nees --- src/libstore/store-api.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ac1d8ee2c..db84ec7a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,6 +786,7 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { + settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto path : paths) { From c182aac98ab6548c16b6686638591ba5b034026a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 25 Feb 2021 17:10:45 +0100 Subject: [PATCH 398/477] Apply @edolstra stylistic suggestions Mostly removing useless comments and adding spaces before `&` Co-authored-by: Eelco Dolstra --- src/libstore/globals.hh | 1 - src/libstore/store-api.cc | 6 +++--- src/libstore/store-api.hh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 25351f55c..a51d9c2f1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,7 +45,6 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -/* MakeError(MissingExperimentalFeature, Error); */ class MissingExperimentalFeature: public Error { public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index db84ec7a2..b7a3f7b11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -789,19 +789,19 @@ std::map copyPaths(ref srcStore, ref dstStor settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; - for (auto path : paths) { + for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - for (auto& realisation : realisations) { + for (auto & realisation : realisations) { dstStore->registerDrvOutput(realisation); } } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might - // not be whithin our control to change that, and we might still want + // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == "ca-derivations") ignoreException(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 63b26422a..742cd18db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,7 +752,7 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const RealisedPath::Set&, + const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From c43f446f4e3a1a8d91560b6ebbcc7d4fbbbf71c4 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 25 Feb 2021 16:58:27 +0100 Subject: [PATCH 399/477] Make `nix copy` work without the ca-derivations flag The experimental feature was by mistake required for `nix copy` to work at oll --- src/libstore/store-api.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b7a3f7b11..77c310988 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,13 +786,14 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto & path : paths) { storePaths.insert(path.path()); - if (auto realisation = std::get_if(&path.raw)) + if (auto realisation = std::get_if(&path.raw)) { + settings.requireExperimentalFeature("ca-derivations"); realisations.insert(*realisation); + } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { From 20ea1de77d9210e145d5ebb1dccd34c856149b2c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 12:35:29 +0100 Subject: [PATCH 400/477] Use std::make_unique --- src/libstore/build/derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 804a79e4c..33c3aeb6e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -259,10 +259,8 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); - auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); - /* Get the derivation. */ - drv = std::unique_ptr(fullDrv); + drv = std::make_unique(worker.store.derivationFromPath(drvPath)); haveDerivation(); } From 453c3a603f4e6fa3f8c706e73f9869bc7f76c640 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 14:55:54 +0100 Subject: [PATCH 401/477] nix flake update: Recreate the lock file This is probably what most people expect it to do. Fixes #3781. There is a new command 'nix flake lock' that has the old behaviour of 'nix flake update', i.e. it just adds missing lock file entries unless overriden using --update-input. --- src/libutil/args.cc | 8 ++++++++ src/libutil/args.hh | 2 ++ src/nix/flake-lock.md | 38 ++++++++++++++++++++++++++++++++++++++ src/nix/flake-update.md | 37 +++++++++---------------------------- src/nix/flake.cc | 34 +++++++++++++++++++++++++++++++++- tests/flakes.sh | 28 ++++++++++++++-------------- 6 files changed, 104 insertions(+), 43 deletions(-) create mode 100644 src/nix/flake-lock.md diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 75eb19d28..afed0670f 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -19,6 +19,14 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +void Args::removeFlag(const std::string & longName) +{ + auto flag = longFlags.find(longName); + assert(flag != longFlags.end()); + if (flag->second->shortName) shortFlags.erase(flag->second->shortName); + longFlags.erase(flag); +} + void Completions::add(std::string completion, std::string description) { assert(description.find('\n') == std::string::npos); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 4721c21df..c08ba8abd 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -140,6 +140,8 @@ public: void addFlag(Flag && flag); + void removeFlag(const std::string & longName); + void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md new file mode 100644 index 000000000..2af0ad81e --- /dev/null +++ b/src/nix/flake-lock.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake lock --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated unless +required by a flag such as `--update-input`. + +Note that every command that operates on a flake will also update the +lock file if needed, and supports the same flags. Therefore, + +```console +# nix flake lock --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index a2ffedd2a..03b50e38e 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,52 +2,33 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: - - ```console - # nix flake update --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - ``` - * Recreate the lock file (i.e. update all inputs) and commit the new lock file: ```console - # nix flake update --recreate-lock-file --commit-lock-file + # nix flake update + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Note that every command that operates on a flake will -also update the lock file if needed, and supports the same -flags. Therefore, +This command recreates the lock file of a flake (`flake.lock`), thus +updating the lock for every mutable input (like `nixpkgs`) to its +current version. This is equivalent to passing `--recreate-lock-file` +to any command that operates on a flake. That is, ```console -# nix flake update --update-input nixpkgs +# nix flake update # nix build ``` is equivalent to: ```console -# nix build --update-input nixpkgs +# nix build --recreate-lock-file ``` -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. - -> **Note** -> -> This command does *not* update locks that are already present unless -> you explicitly ask for it using `--update-input` or -> `--recreate-lock-file`. Thus, if the lock file already has locks for -> every input, then `nix flake update` (without arguments) does -> nothing. - )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b9cde5d6d..2f0c468a8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -104,6 +104,14 @@ struct CmdFlakeUpdate : FlakeCommand return "update flake lock file"; } + CmdFlakeUpdate() + { + /* Remove flags that don't make sense. */ + removeFlag("recreate-lock-file"); + removeFlag("update-input"); + removeFlag("no-update-lock-file"); + } + std::string doc() override { return @@ -113,7 +121,30 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { - /* Use --refresh by default for 'nix flake update'. */ + settings.tarballTtl = 0; + + lockFlags.recreateLockFile = true; + + lockFlake(); + } +}; + +struct CmdFlakeLock : FlakeCommand +{ + std::string description() override + { + return "create missing lock file entries"; + } + + std::string doc() override + { + return + #include "flake-lock.md" + ; + } + + void run(nix::ref store) override + { settings.tarballTtl = 0; lockFlake(); @@ -1006,6 +1037,7 @@ struct CmdFlake : NixMultiCommand CmdFlake() : MultiCommand({ {"update", []() { return make_ref(); }}, + {"lock", []() { return make_ref(); }}, {"info", []() { return make_ref(); }}, {"list-inputs", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, diff --git a/tests/flakes.sh b/tests/flakes.sh index 2b7bcdd68..25ba2ac43 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -232,7 +232,7 @@ nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2 nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh # Updating the flake should not change the lockfile. -nix flake update $flake2Dir +nix flake lock $flake2Dir [[ -z $(git -C $flake2Dir diff master) ]] # Now we should be able to build the flake in pure mode. @@ -354,10 +354,10 @@ nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord nix build -o $TEST_ROOT/result flake4#xyzzy # Test 'nix flake update' and --override-flake. -nix flake update $flake3Dir +nix flake lock $flake3Dir [[ -z $(git -C $flake3Dir diff master) ]] -nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs +nix flake update $flake3Dir --override-flake flake2 nixpkgs [[ ! -z $(git -C $flake3Dir diff master) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore @@ -389,7 +389,7 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Fri, 26 Feb 2021 16:03:39 +0100 Subject: [PATCH 402/477] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31' -> 'github:NixOS/nixpkgs/0e499fde7af3c28d63e9b13636716b86c3162b93' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6fe52fbfd..9867e694b 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1602702596, - "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=", + "lastModified": 1614309161, + "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ad0d20345219790533ebe06571f82ed6b034db31", + "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93", "type": "github" }, "original": { From 14f51880bad5145e73eb150797e757440925913b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:29:30 +0100 Subject: [PATCH 403/477] Update src/build-remote/build-remote.cc --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 228aba35a..1be491603 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -288,7 +288,7 @@ connected: auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { - notice("Missing output %s", outputName); + debug("missing output %s", outputName); assert(result.builtOutputs.count(thisOutputId)); auto newRealisation = result.builtOutputs.at(thisOutputId); missingRealisations.insert(newRealisation); From 17c98e03eac45b3c298567e8a1c04e3d4c4aa0d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:29:37 +0100 Subject: [PATCH 404/477] Update src/build-remote/build-remote.cc --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1be491603..7f3636f6b 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,7 +311,7 @@ connected: localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } - // XXX: Should e done as part of `copyPaths` + // XXX: Should be done as part of `copyPaths` for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty From 076d2b04da72607b67e581572a31db2a220589ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:30:12 +0100 Subject: [PATCH 405/477] Update src/libstore/build/derivation-goal.cc --- src/libstore/build/derivation-goal.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6052b625d..a5622f990 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1164,7 +1164,6 @@ HookReply DerivationGoal::tryBuildHook() // XXX: Does this include known CA outputs? if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); - /* missingPaths.insert(status.known->path); */ } worker_proto::write(worker.store, hook->sink, missingOutputs); } From f54976d77bd144535e9b4844dbdb6bc52eac11fd Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 26 Feb 2021 16:34:33 +0100 Subject: [PATCH 406/477] Simplify the case where the drv is a purely input-addressed one --- src/build-remote/build-remote.cc | 2 +- src/libstore/build/entry-points.cc | 2 +- src/libstore/derivations.cc | 11 +++++++++++ src/libstore/derivations.hh | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 7f3636f6b..736b81542 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -283,7 +283,7 @@ connected: std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 99b3fa070..3a05a022c 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -69,7 +69,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat outputId, Realisation{ outputId, *staticOutput.second} ); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { auto realisation = this->queryRealisation(outputId); if (realisation) result.builtOutputs.insert_or_assign( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6d0742b4f..fe98182bb 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -57,6 +57,17 @@ bool derivationIsFixed(DerivationType dt) { assert(false); } +bool derivationHasKnownOutputPaths(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return true; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; + }; + assert(false); +} + + bool derivationIsImpure(DerivationType dt) { switch (dt) { case DerivationType::InputAddressed: return false; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4e5985fab..061d70f69 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -94,6 +94,11 @@ bool derivationIsFixed(DerivationType); derivation is controlled separately. Never true for non-CA derivations. */ bool derivationIsImpure(DerivationType); +/* Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the closure. + */ +bool derivationHasKnownOutputPaths(DerivationType); + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ From 05cc5a858717c092e1835e2b0fec4c4b1a7fc97e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Feb 2021 06:26:35 +0000 Subject: [PATCH 407/477] Copy {,local-}derivation-goal.{cc,h} Doing this prior to splitting, so we get better diff with default options (e.g. on GitHub). --- src/libstore/build/local-derivation-goal.cc | 3902 +++++++++++++++++++ src/libstore/build/local-derivation-goal.hh | 373 ++ 2 files changed, 4275 insertions(+) create mode 100644 src/libstore/build/local-derivation-goal.cc create mode 100644 src/libstore/build/local-derivation-goal.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc new file mode 100644 index 000000000..924c69fb7 --- /dev/null +++ b/src/libstore/build/local-derivation-goal.cc @@ -0,0 +1,3902 @@ +#include "derivation-goal.hh" +#include "hook-instance.hh" +#include "worker.hh" +#include "builtins.hh" +#include "builtins/buildenv.hh" +#include "references.hh" +#include "finally.hh" +#include "util.hh" +#include "archive.hh" +#include "json.hh" +#include "compression.hh" +#include "daemon.hh" +#include "worker-protocol.hh" +#include "topo-sort.hh" +#include "callback.hh" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_STATVFS +#include +#endif + +/* Includes required for chroot support. */ +#if __linux__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_SECCOMP +#include +#endif +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +#endif + +#if __APPLE__ +#include +#include +#endif + +#include +#include + +#include + +namespace nix { + +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: wrap errors. + ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); + logError(ei); + } + } +} + +const Path DerivationGoal::homeDir = "/homeless-shelter"; + +DerivationGoal::DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(true) + , drvPath(drvPath) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &DerivationGoal::getDerivation; + name = fmt( + "building of '%s' from .drv file", + StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); +} + + +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(false) + , drvPath(drvPath) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + this->drv = std::make_unique(drv); + + state = &DerivationGoal::haveDerivation; + name = fmt( + "building of '%s' from in-memory derivation", + StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(this->drvPath); +} + + +DerivationGoal::~DerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { killChild(); } catch (...) { ignoreException(); } + try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreException(); } +} + + +string DerivationGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); +} + + +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + +void DerivationGoal::killChild() +{ + if (pid != -1) { + worker.childTerminated(this); + + if (buildUser) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser->kill(); + pid.wait(); + } else + pid.kill(); + + assert(pid == -1); + } + + hook.reset(); +} + + +void DerivationGoal::timedOut(Error && ex) +{ + killChild(); + done(BuildResult::TimedOut, ex); +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + for (auto & i : outputs) + if (wantedOutputs.insert(i).second) + needRestart = true; +} + + +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); + + state = &DerivationGoal::loadDerivation; +} + + +void DerivationGoal::loadDerivation() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = std::make_unique(worker.store.derivationFromPath(drvPath)); + + haveDerivation(); +} + + +void DerivationGoal::haveDerivation() +{ + trace("have derivation"); + + if (drv->type() == DerivationType::CAFloating) + settings.requireExperimentalFeature("ca-derivations"); + + retrySubstitution = false; + + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + worker.store.addTempRoot(*i.second.second); + + auto outputHashes = staticOutputHashes(worker.store, *drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + + /* Check what outputs paths are not already valid. */ + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + + /* If they are all valid, then we're done. */ + if (allValid && buildMode == bmNormal) { + done(BuildResult::AlreadyValid); + return; + } + + parsedDrv = std::make_unique(drvPath, *drv); + + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known) { + warn("do not know how to query for unknown floating content-addressed derivation output yet"); + /* Nothing to wait for; tail call */ + return DerivationGoal::gaveUpOnSubstitution(); + } + addWaitee(upcast_goal(worker.makeSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv)))); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstitutionTried(); + else + state = &DerivationGoal::outputsSubstitutionTried; +} + + +void DerivationGoal::outputsSubstitutionTried() +{ + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { + done(BuildResult::TransientFailure, + fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); + return; + } + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + checkPathValidity(); + size_t nrInvalid = 0; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) + nrInvalid++; + } + + if (buildMode == bmNormal && nrInvalid == 0) { + done(BuildResult::Substituted); + return; + } + if (buildMode == bmRepair && nrInvalid == 0) { + repairClosure(); + return; + } + if (buildMode == bmCheck && nrInvalid > 0) + throw Error("some outputs of '%s' are not valid, so checking is not possible", + worker.store.printStorePath(drvPath)); + + /* Nothing to wait for; tail call */ + gaveUpOnSubstitution(); +} + +/* At least one of the output paths could not be + produced using a substitute. So we have to build instead. */ +void DerivationGoal::gaveUpOnSubstitution() +{ + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs.clear(); + + /* The inputs must be built before we can build this goal. */ + if (useDerivation) + for (auto & i : dynamic_cast(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + + for (auto & i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + auto outputs = queryDerivationOutputMap(); + StorePathSet outputClosure; + for (auto & i : outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second, outputClosure); + } + + /* Filter out our own outputs (which we have already checked). */ + for (auto & i : outputs) + outputClosure.erase(i.second); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + StorePathSet inputClosure; + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); + std::map outputsToDrv; + for (auto & i : inputClosure) + if (i.isDerivation()) { + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + for (auto & j : depOutputs) + if (j.second) + outputsToDrv.insert_or_assign(*j.second, i); + } + + /* Check each path (slow!). */ + for (auto & i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printError( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + auto drvPath2 = outputsToDrv.find(i); + if (drvPath2 == outputsToDrv.end()) + addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); + else + addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + } + + if (waitees.empty()) { + done(BuildResult::AlreadyValid); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error("some paths in the output closure of derivation '%s' could not be repaired", + worker.store.printStorePath(drvPath)); + done(BuildResult::AlreadyValid); +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + if (!useDerivation) + throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + if (useDerivation) { + auto & fullDrv = *dynamic_cast(drv.get()); + + if (settings.isExperimentalFeatureEnabled("ca-derivations") && + ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed)) { + /* We are be able to resolve this derivation based on the + now-known results of dependencies. If so, we become a stub goal + aliasing that resolved derivation goal */ + std::optional attempt = fullDrv.tryResolve(worker.store); + assert(attempt); + Derivation drvResolved { *std::move(attempt) }; + + auto pathResolved = writeDerivation(worker.store, drvResolved); + resolvedDrv = drvResolved; + + auto msg = fmt("Resolved derivation: '%s' -> '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved)); + act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, + Logger::Fields { + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved), + }); + + auto resolvedGoal = worker.makeDerivationGoal( + pathResolved, wantedOutputs, buildMode); + addWaitee(resolvedGoal); + + state = &DerivationGoal::resolvedFinished; + return; + } + + for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(drvPath)); + auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + for (auto & j : wantedDepOutputs) { + if (outputs.count(j) > 0) { + auto optRealizedInput = outputs.at(j); + if (!optRealizedInput) + throw Error( + "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.computeFSClosure(*optRealizedInput, inputPaths); + } else + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + } + } + + /* Second, the input sources. */ + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + + debug("added input paths %s", worker.store.showPaths(inputPaths)); + + /* What type of derivation are we building? */ + derivationType = drv->type(); + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + + result = BuildResult(); +} + +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); +} + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + /* Obtain locks on all output paths, if the paths are known a priori. + + The locks are automatically released when we exit this function or Nix + crashes. If we can't acquire the lock, then continue; hopefully some + other goal can start a build, and if not, the main loop will sleep a few + seconds and then retry this goal. */ + PathSet lockFiles; + /* FIXME: Should lock something like the drv itself so we don't build same + CA drv concurrently */ + if (dynamic_cast(&worker.store)) + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + + if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); + worker.waitForAWhile(shared_from_this()); + return; + } + + actLock.reset(); + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + if (buildMode != bmCheck && allValid) { + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); + outputLocks.setDeletion(true); + done(BuildResult::AlreadyValid); + return; + } + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto & [_, status] : initialOutputs) { + if (!status.known || status.known->isValid()) continue; + auto storePath = status.known->path; + debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); + deletePath(worker.store.Store::toRealPath(storePath)); + } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + actLock.reset(); + result.startTime = time(0); // inexact + state = &DerivationGoal::buildDone; + started(); + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + actLock.reset(); + + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + /* Make sure that we are allowed to start a build. */ + if (!dynamic_cast(&worker.store)) { + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + } + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + done(BuildResult::InputRejected, e); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; + + started(); +} + + +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +void replaceValidPath(const Path & storePath, const Path & tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + if (pathExists(storePath)) + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreException(); + } + throw; + } + + deletePath(oldPath); +} + + +MakeError(NotDeterministic, BuildError); + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + sandboxMountNamespace = -1; + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill() : pid.kill(); + + debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); + + result.timesBuilt++; + result.stopTime = time(0); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; + } else + builderOut.readSide = -1; + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + + bool diskFull = false; + + try { + + /* Check the exit status. */ + if (!statusOk(status)) { + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + if (auto localStore = dynamic_cast(&worker.store)) { + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } + + auto msg = fmt("builder for '%s' %s", + yellowtxt(worker.store.printStorePath(drvPath)), + statusToString(status)); + + if (!logger->isVerbose() && !logTail.empty()) { + msg += fmt(";\nlast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } + msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + worker.store.printStorePath(drvPath)); + } + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + registerOutputs(); + + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{worker.store.printStorePath(drvPath)}); + PushActivity pact(act.id); + StorePathSet outputPaths; + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); + } + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (std::string_view data) override { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + + if (buildMode == bmCheck) { + deleteTmpDir(true); + done(BuildResult::Built); + return; + } + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(worker.store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError & e) { + outputLocks.unlock(); + + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; + + else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { + } + + else { + st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; + } + + done(st, e); + return; + } + + done(BuildResult::Built); +} + +void DerivationGoal::resolvedFinished() { + assert(resolvedDrv); + + auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv->outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(initialOutputs.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = worker.store.queryRealisation( + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + ); + // We've just built it, but maybe the build failed, in which case the + // realisation won't be there + if (realisation) { + auto newRealisation = *realisation; + newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; + worker.store.registerDrvOutput(newRealisation); + } else { + // If we don't have a realisation, then it must mean that something + // failed when building the resolved drv + assert(!result.success()); + } + } + + // This is potentially a bit fishy in terms of error reporting. Not sure + // how to do it in a cleaner way + amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); +} + +HookReply DerivationGoal::tryBuildHook() +{ + if (!worker.tryBuildHook || !useDerivation) return rpDecline; + + if (!worker.hook) + worker.hook = std::make_unique(); + + try { + + /* Send the request to the hook. */ + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << worker.store.printStorePath(drvPath) + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + auto s = [&]() { + try { + return readLine(worker.hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response from the build hook"); + throw e; + } + }(); + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + else { + s += "\n"; + writeToStderr(s); + } + } + + debug("hook reply is '%1%'", reply); + + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; + else if (reply != "accept") + throw Error("bad hook reply '%s'", reply); + + } catch (SysError & e) { + if (e.errNo == EPIPE) { + printError( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); + worker.hook = 0; + return rpDecline; + } else + throw; + } + + hook = std::move(worker.hook); + + try { + machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw e; + } + + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + worker_proto::write(worker.store, hook->sink, inputPaths); + + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + { + StringSet missingOutputs; + for (auto & [outputName, status] : initialOutputs) { + // XXX: Does this include known CA outputs? + if (buildMode != bmCheck && status.known && status.known->isValid()) continue; + missingOutputs.insert(outputName); + } + worker_proto::write(worker.store, hook->sink, missingOutputs); + } + + hook->sink = FdSink(); + hook->toHook.writeSide = -1; + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + set fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); + + return rpAccept; +} + + +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->runChild(); + return 1; +} + + +StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) +{ + StorePathSet paths; + + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); + + worker.store.computeFSClosure({storePath}, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = paths; + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputsAndOptPaths(worker.store)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + worker.store.computeFSClosure(*k.second.second, paths); + } + } + } + + return paths; +} + +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } + }); +} + + +void linkOrCopy(const Path & from, const Path & to) +{ + if (link(from.c_str(), to.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum link count on a + file (e.g. 32000 of ext3), which is quite possible after a + 'nix-store --optimise'. FIXME: actually, why don't we just + bind-mount in this case? + + It can also fail with EPERM in BeegFS v7 and earlier versions + which don't allow hard-links to other directories */ + if (errno != EMLINK && errno != EPERM) + throw SysError("linking '%s' to '%s'", to, from); + copyPath(from, to); + } +} + + +void DerivationGoal::startBuilder() +{ + /* Right platform? */ + if (!parsedDrv->canBuildLocally(worker.store)) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + worker.store.printStorePath(drvPath), + settings.thisSystem, + concatStringsSep(", ", worker.store.systemFeatures)); + + if (drv->isBuiltin()) + preloadNSS(); + +#if __APPLE__ + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); +#endif + + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = !(derivationIsImpure(derivationType)) && !noChroot; + } + + if (auto localStoreP = dynamic_cast(&worker.store)) { + auto & localStore = *localStoreP; + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + } + + /* Create a temporary directory where the build will take + place. */ + tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); + + chownToBuilder(tmpDir); + + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* A non-removed corrupted path needs to be stored here, too */ + if (buildMode == bmRepair && !status.known->isValid()) + redirectedBadOutputs.insert(status.known->path); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(worker.store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph").value_or(""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto storePathS = *i++; + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + auto storePath = worker.store.toStorePath(storePathS).first; + + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences({storePath}), false, false)); + } + } + + if (useChroot) { + + /* Allow a user-configurable set of directories from the + host file system. */ + dirsInChroot.clear(); + + for (auto i : settings.sandboxPaths.get()) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == string::npos) + dirsInChroot[i] = {i, optional}; + else + dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + } + dirsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + StorePathSet closure; + for (auto & i : dirsInChroot) + try { + if (worker.store.isInStore(i.second.source)) + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); + } catch (InvalidPath & e) { + } catch (Error & e) { + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; + } + for (auto & i : closure) { + auto p = worker.store.printStorePath(i); + dirsInChroot.insert_or_assign(p, p); + } + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + worker.store.printStorePath(drvPath), i); + + dirsInChroot[i] = i; + } + +#if __linux__ + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared(chrootRootDir); + + printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + + if (mkdir(chrootRootDir.c_str(), 0750) == -1) + throw SysError("cannot create '%1%'", chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", sandboxGid())); + + /* Create /etc/hosts with localhost entry. */ + if (!(derivationIsImpure(derivationType))) + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); + + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + Path r = worker.store.toRealPath(p); + if (S_ISDIR(lstat(r).st_mode)) + dirsInChroot.insert_or_assign(p, r); + else + linkOrCopy(r, chrootRootDir + p); + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.dirsInChroot + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + /* If the name isn't known a priori (i.e. floating + content-addressed derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + } + +#elif __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ +#else + throw Error("sandboxing builds is not supported on this platform"); +#endif + } + + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); + + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, format("executing pre-build hook '%1%'") + % settings.preBuildHook); + auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : + Strings({ worker.store.printStorePath(drvPath) }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos - lastPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error("unknown pre-build hook command '%1%'", line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == string::npos) + dirsInChroot[line] = line; + else + dirsInChroot[string(line, 0, p)] = string(line, p + 1); + } + } + } + } + + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) + startDaemon(); + + /* Run the builder. */ + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } +#if __APPLE__ + else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } +#endif + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + result.startTime = time(0); + + /* Fork a child to build the package. */ + ProcessOptions options; + +#if __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + if (!(derivationIsImpure(derivationType))) + privateNetwork = true; + + userNamespaceSync.create(); + + options.allowVfork = false; + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + static bool userNamespacesEnabled = + pathExists(maxUserNamespaces) + && trim(readFile(maxUserNamespaces)) != "0"; + + usingUserNamespace = userNamespacesEnabled; + + Pid helper = startProcess([&]() { + + /* Drop additional groups here because we can't do it + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ + if (getuid() == 0 && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) + flags |= CLONE_NEWNET; + if (usingUserNamespace) + flags |= CLONE_NEWUSER; + + pid_t child = clone(childEntry, stack + stackSize, flags, this); + if (child == -1 && errno == EINVAL) { + /* Fallback for Linux < 2.13 where CLONE_NEWPID and + CLONE_PARENT are not allowed together. */ + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unprivileged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + usingUserNamespace = false; + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); + if (child == -1) throw SysError("cloning builder process"); + + writeFull(builderOut.writeSide.get(), + fmt("%d %d\n", usingUserNamespace, child)); + _exit(0); + }, options); + + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) + throw Error("unable to start build process"); + + userNamespaceSync.readSide = -1; + + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + + auto ss = tokenizeString>(readLine(builderOut.readSide.get())); + assert(ss.size() == 2); + usingUserNamespace = ss[0] == "1"; + pid = string2Int(ss[1]).value(); + + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + fmt("%d %d 1", sandboxUid(), hostUid)); + + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + fmt("%d %d 1", sandboxGid(), hostGid)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + } + + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + + /* Save the mount namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + + /* Signal the builder that we've updated its user namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + + } else +#endif + { + fallback: + options.allowVfork = !buildUser && !drv->isBuiltin(); + pid = startProcess([&]() { + runChild(); + }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = -1; + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + + /* Check if setting up the build environment failed. */ + std::vector msgs; + while (true) { + string msg = [&]() { + try { + return readLine(builderOut.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + concatStringsSep("|", msgs)); + throw e; + } + }(); + if (string(msg, 0, 1) == "\2") break; + if (string(msg, 0, 1) == "\1") { + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; + } + debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); + } +} + + +void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + + StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); + for (auto & i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(htSHA256, i.first); + string fn = ".attr-" + hash.to_string(Base32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (derivationIsImpure(derivationType)) { + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i).value_or(""); + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + +void DerivationGoal::writeStructuredAttrs() +{ + auto structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + } + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(worker.store.parseStorePath(p.get())); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); +} + +struct RestrictedStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; + +/* A wrapper around LocalStore that only allows building/querying of + paths that are in the input closures of the build or were added via + recursive Nix calls. */ +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore +{ + ref next; + + DerivationGoal & goal; + + RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) + { } + + Path getRealStoreDir() override + { return next->realStoreDir; } + + std::string getUri() override + { return next->getUri(); } + + StorePathSet queryAllValidPaths() override + { + StorePathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); + return paths; + } + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override + { + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.reset(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); + }; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override + { } + + std::map> queryPartialDerivationOutputMap(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); + return next->queryPartialDerivationOutputMap(path); + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { throw Error("queryPathFromHashPart"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + { throw Error("addToStore"); } + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override + { + next->addToStore(info, narSource, repair, checkSigs); + goal.addDependency(info.path); + } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override + { + auto path = next->addTextToStore(name, s, references, repair); + goal.addDependency(path); + return path; + } + + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); + LocalFSStore::narFromPath(path, sink); + } + + void ensurePath(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); + /* Nothing to be done; 'path' must already be valid. */ + } + + void registerDrvOutput(const Realisation & info) override + // XXX: This should probably be allowed as a no-op if the realisation + // corresponds to an allowed derivation + { throw Error("registerDrvOutput"); } + + std::optional queryRealisation(const DrvOutput & id) override + // XXX: This should probably be allowed if the realisation corresponds to + // an allowed derivation + { throw Error("queryRealisation"); } + + void buildPaths(const std::vector & paths, BuildMode buildMode) override + { + if (buildMode != bmNormal) throw Error("unsupported build mode"); + + StorePathSet newPaths; + + for (auto & path : paths) { + if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + } + + next->buildPaths(paths, buildMode); + + for (auto & path : paths) { + if (!path.path.isDerivation()) continue; + auto outputs = next->queryDerivationOutputMap(path.path); + for (auto & output : outputs) + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second); + } + + StorePathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + void addTempRoot(const StorePath & path) override + { } + + void addIndirectRoot(const Path & path) override + { } + + Roots findRoots(bool censor) override + { return Roots(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { } + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override + { unsupported("addSignatures"); } + + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override + { + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + std::vector allowed; + for (auto & path : targets) { + if (goal.isAllowed(path.path)) + allowed.emplace_back(path); + else + unknown.insert(path.path); + } + + next->queryMissing(allowed, willBuild, willSubstitute, + unknown, downloadSize, narSize); + } +}; + + +void DerivationGoal::startDaemon() +{ + settings.requireExperimentalFeature("recursive-nix"); + + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = worker.store.storeDir; + if (auto localStore = dynamic_cast(&worker.store)) + params["root"] = localStore->rootDir; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = make_ref(params, + ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR) continue; + if (errno == EINVAL) break; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + debug("received daemon connection"); + + auto workerThread = std::thread([store, remote{std::move(remote)}]() { + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, + [&](Store & store) { store.createUser("nobody", 65535); }); + debug("terminated daemon connection"); + } catch (SysError &) { + ignoreException(); + } + }); + + daemonWorkerThreads.push_back(std::move(workerThread)); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationGoal::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) + throw SysError("shutting down daemon socket"); + + if (daemonThread.joinable()) + daemonThread.join(); + + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + + daemonSocket = -1; +} + + +void DerivationGoal::addDependency(const StorePath & path) +{ + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); + + #if __linux__ + + Path source = worker.store.Store::toRealPath(path); + Path target = chrootRootDir + worker.store.printStorePath(path); + debug("bind-mounting %s -> %s", target, source); + + if (pathExists(target)) + throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + + auto st = lstat(source); + + if (S_ISDIR(st.st_mode)) { + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + createDirs(target); + + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + + } else + linkOrCopy(source, target); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + worker.store.printStorePath(path)); + #endif + + } +} + + +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", path); +} + + +void setupSeccomp() +{ +#if __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + + +void DerivationGoal::runChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + try { /* child */ + + commonChildInit(builderOut); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + + bool setUser = true; + + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + netrcData = readFile(settings.netrcFile); + } catch (SysError &) { } + +#if __linux__ + if (useChroot) { + + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (privateNetwork) { + + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount '%1%'", chrootRootDir); + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (derivationIsImpure(derivationType)) { + ss.push_back("/etc/resolv.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + ss.push_back("/etc/services"); + ss.push_back("/etc/hosts"); + if (pathExists("/var/run/nscd/socket")) + ss.push_back("/var/run/nscd/socket"); + } + + for (auto & i : ss) dirsInChroot.emplace(i, i); + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") + && !dirsInChroot.count("/dev/pts")) + { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError("cannot change directory to '%1%'", chrootRootDir); + + if (mkdir("real-root", 0) == -1) + throw SysError("cannot create real-root directory"); + + if (pivot_root(".", "real-root") == -1) + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); + + if (chroot(".") == -1) + throw SysError("cannot change root directory to '%1%'", chrootRootDir); + + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); + + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid()) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid()) == -1) + throw SysError("setuid failed"); + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv->platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set i686-linux personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto & i : env) + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + Strings args; + + const char *builder = "invalid"; + + if (drv->isBuiltin()) { + ; + } +#if __APPLE__ + else { + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + dirsInChroot[p] = p; + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + + if (derivationIsImpure(derivationType)) + sandboxProfile += "(import \"sandbox-network.sb\")\n"; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : dirsInChroot) { + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; + throw SysError("getting attributes of path '%s", path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = tmpDir + "/.sandbox.sb"; + + writeFile(sandboxFile, sandboxProfile); + + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); + } else { + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); + } + } +#else + else { + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); + } +#endif + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, string("\2\n")); + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(*logger); + + BasicDerivation & drv2(*drv); + for (auto & e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); + + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); + else + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, e.what() + std::string("\n")); + _exit(1); + } + } + +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else + execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif + + throw SysError("executing '%1%'", drv->builder); + + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + _exit(1); + } +} + + +void DerivationGoal::registerOutputs() +{ + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. + + We can only early return when the outputs are known a priori. For + floating content-addressed derivations this isn't the case. + */ + if (hook) { + bool allValid = true; + for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!outputPath || !worker.store.isValidPath(*outputPath)) + allValid = false; + else + finalOutputs.insert_or_assign(outputName, *outputPath); + } + if (allValid) return; + } + + std::map infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; + + std::exception_ptr delayedException; + + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); + for (auto & p : addedPaths) referenceablePaths.insert(p); + + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : worker.store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map> outputReferencesIfUnregistered; + std::map outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto & initialInfo = initialOutputs.at(outputName); + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; + } + + struct stat st; + if (lstat(actualPath.c_str(), &st) == -1) { + if (errno == ENOENT) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); + throw SysError("getting attributes of path '%s'", actualPath); + } + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); +#endif + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + auto references = worker.store.parseStorePathSet( + scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); + + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } + + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](AlreadyRegistered _) { return StringSet {}; }, + [&](PerhapsNeedToRegister refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, outputReferencesIfUnregistered.at(name)); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + worker.store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + for (auto & outputName : sortedOutputNames) { + auto output = drv->outputs.at(outputName); + auto & scratchPath = scratchOutputs.at(outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + }; + + std::optional referencesOpt = std::visit(overloaded { + [&](AlreadyRegistered skippedFinalPath) -> std::optional { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](PerhapsNeedToRegister r) -> std::optional { + return r.refs; + }, + }, outputReferencesIfUnregistered.at(outputName)); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&]() { + /* Apply hash rewriting if necessary. */ + if (!outputRewrites.empty()) { + warn("rewriting hashes in '%1%'; cross fingers", actualPath); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> std::pair { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + std::pair res { + false, + {}, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == scratchPath) + res.first = true; + else if (outputRewrites.count(origHash) == 0) + res.second.insert(r); + else { + std::string newRef = outputRewrites.at(origHash); + newRef += '-'; + newRef += name; + res.second.insert(StorePath { newRef }); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto & st = outputStats.at(outputName); + if (outputHash.method == FileIngestionMethod::Flat) { + /* The output path should be a regular file without execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + actualPath); + } + rewriteOutput(); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath.hashPart() }; + HashModuloSink caSink { outputHash.hashType, oldHashPart }; + switch (outputHash.method) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + auto got = caSink.finish().first; + auto refs = rewriteRefs(); + HashModuloSink narSink { htSHA256, oldHashPart }; + dumpPath(actualPath, narSink); + auto narHashAndSize = narSink.finish(); + ValidPathInfo newInfo0 { + worker.store.makeFixedOutputPath( + outputHash.method, + got, + outputPathName(drv->name, outputName), + refs.second, + refs.first), + narHashAndSize.first, + }; + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + if (scratchPath != newInfo0.path) { + // Also rewrite the output path + auto source = sinkToSource([&](Sink & nextSink) { + StringSink sink; + dumpPath(actualPath, sink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + rsink2(*sink.s); + rsink2.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + } + + assert(newInfo0.ca); + return newInfo0; + }; + + ValidPathInfo newInfo = std::visit(overloaded { + [&](DerivationOutputInputAddressed output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath.hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(); + auto narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + [&](DerivationOutputCAFixed dof) { + auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + .method = dof.hash.method, + .hashType = dof.hash.hash.type, + }); + + /* Check wanted hash */ + Hash & wanted = dof.hash.hash; + assert(newInfo0.ca); + auto got = getContentAddressHash(*newInfo0.ca); + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + worker.store.printStorePath(drvPath), + wanted.to_string(SRI, true), + got.to_string(SRI, true))); + } + return newInfo0; + }, + [&](DerivationOutputCAFloating dof) { + return newInfoFromCA(dof); + }, + [&](DerivationOutputDeferred) { + // No derivation should reach that point without having been + // rewritten first + assert(false); + // Ugly, but the compiler insists on having this return a value + // of type `ValidPathInfo` despite the `assert(false)`, so + // let's provide it + return *(ValidPathInfo*)0; + }, + }, output.output); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = worker.store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + auto optFixedPath = output.path(worker.store, drv->name, outputName); + if (!optFixedPath || + worker.store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + } + + /* Move files, if needed */ + if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); + actualPath = worker.store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (worker.store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = worker.store.toRealPath(finalDestPath); + movePath(actualPath, destPath); + actualPath = destPath; + } + } + + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); + auto & localStore = *localStoreP; + + if (buildMode == bmCheck) { + + if (!worker.store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { + auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); + deletePath(dst); + movePath(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); + + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); + } else + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); + } + + /* Since we verified the build, it's now ultimately trusted. */ + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (auto & i : inputPaths) { + auto j = references.find(i); + if (j == references.end()) + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); + else + debug("referenced input: '%1%'", worker.store.printStorePath(i)); + } + + if (curRound == nrRounds) { + localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); + } + + newInfo.deriver = drvPath; + newInfo.ultimate = true; + localStore.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + localStore.registerValidPaths({{newInfo.path, newInfo}}); + + infos.emplace(outputName, std::move(newInfo)); + } + + if (buildMode == bmCheck) return; + + /* Apply output checks. */ + checkOutputs(infos); + + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + result.isNonDeterministic = true; + Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; + bool prevExists = keepPreviousRound && pathExists(prev); + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) + : hintfmt("output '%s' of '%s' differs from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, worker.store.printStorePath(i->second.path), + worker.store.printStorePath(drvPath), tmpDir); + + if (settings.enforceDeterminism) + throw NotDeterministic(hint); + + printError(hint); + + curRound = nrRounds; // we know enough, bail out early + } + } + + /* If this is the first round of several, then move the output out of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { + for (auto & [_, outputStorePath] : finalOutputs) { + auto path = worker.store.printStorePath(outputStorePath); + Path prev = path + checkSuffix; + deletePath(prev); + Path dst = path + checkSuffix; + if (rename(path.c_str(), dst.c_str())) + throw SysError("renaming '%s' to '%s'", path, dst); + } + } + + if (curRound < nrRounds) { + prevInfos = std::move(infos); + return; + } + + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & [_, outputStorePath] : finalOutputs) { + Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; + deletePath(prev); + } + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); + auto & localStore = *localStoreP; + + ValidPathInfos infos2; + for (auto & [outputName, newInfo] : infos) { + infos2.insert_or_assign(newInfo.path, newInfo); + } + localStore.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto& [outputName, newInfo] : infos) + worker.store.registerDrvOutput(Realisation{ + .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, + .outPath = newInfo.path}); + } +} + + +void DerivationGoal::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) + { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(worker.store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) + { + if (!value) return; + + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : *value) { + if (worker.store.isStorePath(i)) + spec.insert(worker.store.parseStorePath(i)); + else if (finalOutputs.count(i)) + spec.insert(finalOutputs.at(i)); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); + } + + auto used = recursive + ? getClosure(info.path).first + : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += worker.store.printStorePath(i); + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + worker.store.printStorePath(info.path), badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get(); + + auto get = [&](const std::string & name) -> std::optional { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); + res.push_back(j->get()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + +Path DerivationGoal::openLogFile() +{ + logSize = 0; + + if (!settings.keepLog) return ""; + + auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); + + /* Create a log file. */ + Path logDir; + if (auto localStore = dynamic_cast(&worker.store)) + logDir = localStore->logDir; + else + logDir = settings.nixLogDir; + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); + createDirs(dir); + + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); + + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); + + logFileSink = std::make_shared(fdLogFile.get()); + + if (settings.compressLog) + logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; + + return logFileName; +} + + +void DerivationGoal::closeLogFile() +{ + auto logSink2 = std::dynamic_pointer_cast(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; + fdLogFile = -1; +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError("note: keeping build directory '%s'", tmpDir); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::handleChildOutput(int fd, const string & data) +{ + if ((hook && fd == hook->builderOut.readSide.get()) || + (!hook && fd == builderOut.readSide.get())) + { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + killChild(); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); + return; + } + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); + } + + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } +} + + +void DerivationGoal::handleEOF(int fd) +{ + if (!currentLogLine.empty()) flushLine(); + worker.wakeUp(shared_from_this()); +} + + +void DerivationGoal::flushLine() +{ + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; + + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); + + act->result(resBuildLogLine, currentLogLine); + } + + currentLogLine = ""; + currentLogLinePos = 0; +} + + +std::map> DerivationGoal::queryPartialDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + std::map> res; + for (auto & [name, output] : drv->outputs) + res.insert_or_assign(name, output.path(worker.store, drv->name, name)); + return res; + } else { + return worker.store.queryPartialDerivationOutputMap(drvPath); + } +} + +OutputPathMap DerivationGoal::queryDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + OutputPathMap res; + for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) + res.insert_or_assign(name, *output.second); + return res; + } else { + return worker.store.queryDerivationOutputMap(drvPath); + } +} + + +void DerivationGoal::checkPathValidity() +{ + bool checkHash = buildMode == bmRepair; + for (auto & i : queryPartialDerivationOutputMap()) { + InitialOutput & info = initialOutputs.at(i.first); + info.wanted = wantOutput(i.first, wantedOutputs); + if (i.second) { + auto outputPath = *i.second; + info.known = { + .path = outputPath, + .status = !worker.store.isValidPath(outputPath) + ? PathStatus::Absent + : !checkHash || worker.pathContentsGood(outputPath) + ? PathStatus::Valid + : PathStatus::Corrupt, + }; + } + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (auto real = worker.store.queryRealisation( + DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { + info.known = { + .path = real->outPath, + .status = PathStatus::Valid, + }; + } + } + } +} + + +StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + Hash(htSHA256), outputPathName(drv->name, outputName)); +} + + +StorePath DerivationGoal::makeFallbackPath(const StorePath & path) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + Hash(htSHA256), path.name()); +} + + +void DerivationGoal::done(BuildResult::Status status, std::optional ex) +{ + result.status = status; + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); + if (result.status == BuildResult::TimedOut) + worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) + worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) + worker.failedBuilds++; + } + + worker.updateProgress(); +} + + +} diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh new file mode 100644 index 000000000..6dc164922 --- /dev/null +++ b/src/libstore/build/local-derivation-goal.hh @@ -0,0 +1,373 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "lock.hh" +#include "local-store.hh" +#include "goal.hh" + +namespace nix { + +using std::map; + +struct HookInstance; + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + +/* Unless we are repairing, we don't both to test validity and just assume it, + so the choices are `Absent` or `Valid`. */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus { + StorePath path; + PathStatus status; + /* Valid in the store, and additionally non-corrupt if we are repairing */ + bool isValid() const { + return status == PathStatus::Valid; + } + /* Merely present, allowed to be corrupt */ + bool isPresent() const { + return status == PathStatus::Corrupt + || status == PathStatus::Valid; + } +}; + +struct InitialOutput { + bool wanted; + Hash outputHash; + std::optional known; +}; + +struct DerivationGoal : public Goal +{ + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + + /* The path of the derivation. */ + StorePath drvPath; + + /* The path of the corresponding resolved derivation */ + std::optional resolvedDrv; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart = false; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + std::unique_ptr drv; + + std::unique_ptr parsedDrv; + + /* The remainder is state held during the build. */ + + /* Locks on (fixed) output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + StorePathSet inputPaths; + + std::map initialOutputs; + + /* User selected for running the builder. */ + std::unique_ptr buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + std::shared_ptr logFileSink, logSink; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* The most recent log lines. */ + std::list logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + + std::string currentHookLine; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* Pipe for synchronising updates to the builder namespaces. */ + Pipe userNamespaceSync; + + /* The mount namespace of the builder, used to add additional + paths to the sandbox as a result of recursive Nix calls. */ + AutoCloseFD sandboxMountNamespace; + + /* On Linux, whether we're doing the build in its own user + namespace. */ + bool usingUserNamespace = true; + + /* The build hook. */ + std::unique_ptr hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot = false; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + std::shared_ptr autoDelChroot; + + /* The sort of derivation we are building. */ + DerivationType derivationType; + + /* Whether to run the build in a private network namespace. */ + bool privateNetwork = false; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; + + typedef map Environment; + Environment env; + +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; +#endif + + /* Hash rewriting. */ + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /* The outputs paths used during the build. + + - Input-addressed derivations or fixed content-addressed outputs are + sometimes built when some of their outputs already exist, and can not + be hidden via sandboxing. We use temporary locations instead and + rewrite after the build. Otherwise the regular predetermined paths are + put here. + + - Floating content-addressed derivations do not know their final build + output paths until the outputs are hashed, so random locations are + used, and then renamed. The randomness helps guard against hidden + self-references. + */ + OutputPathMap scratchOutputs; + + /* The final output paths of the build. + + - For input-addressed derivations, always the precomputed paths + + - For content-addressed derivations, calcuated from whatever the hash + ends up being. (Note that fixed outputs derivations that produce the + "wrong" output still install that data under its true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + /* If we're repairing without a chroot, there may be outputs that + are valid but corrupt. So we redirect these outputs to + temporary paths. */ + StorePathSet redirectedBadOutputs; + + BuildResult result; + + /* The current round, if we're building multiple times. */ + size_t curRound = 1; + + size_t nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + std::map prevInfos; + + uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } + + const static Path homeDir; + + std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; + + std::unique_ptr act; + + /* Activity that denotes waiting for a lock. */ + std::unique_ptr actLock; + + std::map builderActivities; + + /* The remote machine on which we're building. */ + std::string machineName; + + /* The recursive Nix daemon socket. */ + AutoCloseFD daemonSocket; + + /* The daemon main thread. */ + std::thread daemonThread; + + /* The daemon worker threads. */ + std::vector daemonWorkerThreads; + + /* Paths that were added via recursive Nix calls. */ + StorePathSet addedPaths; + + /* Recursive Nix calls are only allowed to build or realize paths + in the original input closure or added via a recursive Nix call + (so e.g. you can't do 'nix-store -r /nix/store/' where + /nix/store/ is some arbitrary path in a binary cache). */ + bool isAllowed(const StorePath & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + + friend struct RestrictedStore; + + DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + ~DerivationGoal(); + + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + + void timedOut(Error && ex) override; + + string key() override; + + void work() override; + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + + BuildResult getResult() { return result; } + + /* The states. */ + void getDerivation(); + void loadDerivation(); + void haveDerivation(); + void outputsSubstitutionTried(); + void gaveUpOnSubstitution(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void tryLocalBuild(); + void buildDone(); + + void resolvedFinished(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Setup tmp dir location. */ + void initTmpDir(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + void startDaemon(); + + void stopDaemon(); + + /* Add 'path' to the set of paths that may be referenced by the + outputs, and make it appear in the sandbox. */ + void addDependency(const StorePath & path); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path & path); + + /* Run the builder's process. */ + void runChild(); + + /* Check that the derivation outputs all exist and register them + as valid. */ + void registerOutputs(); + + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map & outputs); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + void flushLine(); + + /* Wrappers around the corresponding Store methods that first consult the + derivation. This is currently needed because when there is no drv file + there also is no DB entry. */ + std::map> queryPartialDerivationOutputMap(); + OutputPathMap queryDerivationOutputMap(); + + /* Return the set of (in)valid paths. */ + void checkPathValidity(); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + /* Create alternative path calculated from but distinct from the + input, so we can avoid overwriting outputs (or other store paths) + that already exist. */ + StorePath makeFallbackPath(const StorePath & path); + /* Make a path to another based on the output name along with the + derivation hash. */ + /* FIXME add option to randomize, so we can audit whether our + rewrites caught everything */ + StorePath makeFallbackPath(std::string_view outputName); + + void repairClosure(); + + void started(); + + void done( + BuildResult::Status status, + std::optional ex = {}); + + StorePathSet exportReferences(const StorePathSet & storePaths); +}; + +} From 68f4c728eca33f115f90e3f924c9081a4cd59896 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 15:20:33 +0000 Subject: [PATCH 408/477] Split {,local-}derivation-goal.{cc,hh} This separates the scheduling logic (including simple hook pathway) from the local-store needing code. This should be the final split for now. I'm reasonably happy with how it's turning out, even before I'm done moving code into `local-derivation-goal`. Benefits: 1. This will help "witness" that the hook case is indeed a lot simpler, and also compensate for the increased complexity that comes from content-addressed derivation outputs. 2. It also moves us ever so slightly towards a world where we could use off-the-shelf storage or sandboxing, since `local-derivation-goal` would be gutted in those cases, but `derivation-goal` should remain nearly the same. The new `#if 0` in the new files will be deleted in the following commit. I keep it here so if it turns out more stuff can be moved over, it's easy to do so in a way that preserves ordering --- and thus prevents conflicts. N.B. ```sh git diff HEAD^^ --color-moved --find-copies-harder --patience --stat ``` makes nicer output. --- src/libstore/build/derivation-goal.cc | 2741 +------------------ src/libstore/build/derivation-goal.hh | 186 +- src/libstore/build/entry-points.cc | 1 + src/libstore/build/local-derivation-goal.cc | 330 +-- src/libstore/build/local-derivation-goal.hh | 100 +- src/libstore/build/worker.cc | 14 +- src/libstore/local-store.hh | 2 +- 7 files changed, 329 insertions(+), 3045 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 924c69fb7..c29237f5c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -9,10 +9,10 @@ #include "archive.hh" #include "json.hh" #include "compression.hh" -#include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" +#include "local-store.hh" // TODO remove, along with remaining downcasts #include #include @@ -62,40 +62,6 @@ namespace nix { -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - // FIXME: wrap errors. - ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); - logError(ei); - } - } -} - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -144,9 +110,6 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { closeLogFile(); } catch (...) { ignoreException(); } } @@ -161,38 +124,8 @@ string DerivationGoal::key() } -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - void DerivationGoal::killChild() { - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - hook.reset(); } @@ -697,64 +630,10 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - /* Make sure that we are allowed to start a build. */ - if (!dynamic_cast(&worker.store)) { - throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - } - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); } @@ -811,25 +690,63 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) } -MakeError(NotDeterministic, BuildError); +int DerivationGoal::getChildStatus() +{ + return hook->pid.kill(); +} + + +void DerivationGoal::closeReadPipes() +{ + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; +} + + +void DerivationGoal::cleanupHookFinally() +{ +} + + +void DerivationGoal::cleanupPreChildKill() +{ +} + + +void DerivationGoal::cleanupPostChildKill() +{ +} + + +bool DerivationGoal::cleanupDecideWhetherDiskFull() +{ + return false; +} + + +void DerivationGoal::cleanupPostOutputsRegisteredModeCheck() +{ +} + + +void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() +{ +} void DerivationGoal::buildDone() { trace("build done"); - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); + Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - sandboxMountNamespace = -1; + cleanupPreChildKill(); /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); + int status = getChildStatus(); debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); @@ -840,24 +757,12 @@ void DerivationGoal::buildDone() worker.childTerminated(this); /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; + closeReadPipes(); /* Close the log file. */ closeLogFile(); - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); + cleanupPostChildKill(); bool diskFull = false; @@ -866,36 +771,7 @@ void DerivationGoal::buildDone() /* Check the exit status. */ if (!statusOk(status)) { - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - if (auto localStore = dynamic_cast(&worker.store)) { - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } + diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", yellowtxt(worker.store.printStorePath(drvPath)), @@ -975,19 +851,12 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeCheck(); done(BuildResult::Built); return; } - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { @@ -1171,13 +1040,6 @@ HookReply DerivationGoal::tryBuildHook() } -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { StorePathSet paths; @@ -1213,1769 +1075,6 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) return paths; } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (auto localStoreP = dynamic_cast(&worker.store)) { - auto & localStore = *localStoreP; - if (localStore.storeDir != localStore.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - dirsInChroot.clear(); - - for (auto i : settings.sandboxPaths.get()) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int(ss[1]).value(); - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Now that we now the sandbox uid, we can write - /etc/passwd. */ - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - std::vector msgs; - while (true) { - string msg = [&]() { - try { - return readLine(builderOut.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", - concatStringsSep("|", msgs)); - throw e; - } - }(); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - msgs.push_back(std::move(msg)); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : virtual LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void registerDrvOutput(const Realisation & info) override - // XXX: This should probably be allowed as a no-op if the realisation - // corresponds to an allowed derivation - { throw Error("registerDrvOutput"); } - - std::optional queryRealisation(const DrvOutput & id) override - // XXX: This should probably be allowed if the realisation corresponds to - // an allowed derivation - { throw Error("queryRealisation"); } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto localStore = dynamic_cast(&worker.store)) - params["root"] = localStore->rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - -#if __APPLE__ - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) - throw SysError("failed to initialize builder"); - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) - throw SysError("failed to initialize builder"); - - if (drv->platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } - - posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#else - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#endif - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - void DerivationGoal::registerOutputs() { @@ -2986,698 +1085,23 @@ void DerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - if (hook) { - bool allValid = true; - for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!outputPath || !worker.store.isValidPath(*outputPath)) - allValid = false; - else - finalOutputs.insert_or_assign(outputName, *outputPath); - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); + for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!wantOutput(outputName, wantedOutputs)) continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) + if (!optOutputPath) throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif + "output '%s' from derivation '%s' does not have a known output path", + outputName, worker.store.printStorePath(drvPath)); + auto & outputPath = *optOutputPath; + if (!worker.store.isValidPath(outputPath)) + throw BuildError( + "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid", + outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath)); - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - warn("rewriting hashes in '%1%'; cross fingers", actualPath); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2(*sink.s); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); - } - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - [&](DerivationOutputDeferred) { - // No derivation should reach that point without having been - // rewritten first - assert(false); - // Ugly, but the compiler insists on having this return a value - // of type `ValidPathInfo` despite the `assert(false)`, so - // let's provide it - return *(ValidPathInfo*)0; - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; - - if (buildMode == bmCheck) { - - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - localStore.signPathInfo(oldInfo); - localStore.registerValidPaths({{oldInfo.path, oldInfo}}); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - localStore.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - localStore.registerValidPaths({{newInfo.path, newInfo}}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; - - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.insert_or_assign(newInfo.path, newInfo); - } - localStore.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) - worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, - .outPath = newInfo.path}); + finalOutputs.insert_or_assign(outputName, outputPath); } } - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - Path DerivationGoal::openLogFile() { logSize = 0; @@ -3722,26 +1146,15 @@ void DerivationGoal::closeLogFile() } -void DerivationGoal::deleteTmpDir(bool force) +bool DerivationGoal::isReadDesc(int fd) { - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } + return fd == hook->builderOut.readSide.get(); } void DerivationGoal::handleChildOutput(int fd, const string & data) { - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) + if (isReadDesc(fd)) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { @@ -3857,22 +1270,6 @@ void DerivationGoal::checkPathValidity() } -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 6dc164922..c85bcd84f 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,7 +2,8 @@ #include "parsed-derivations.hh" #include "lock.hh" -#include "local-store.hh" +#include "store-api.hh" +#include "pathlocks.hh" #include "goal.hh" namespace nix { @@ -79,18 +80,6 @@ struct DerivationGoal : public Goal std::map initialOutputs; - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - /* File descriptor for the log file. */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; @@ -106,79 +95,15 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - /* The build hook. */ std::unique_ptr hook; - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - /* The sort of derivation we are building. */ DerivationType derivationType; - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - typedef void (DerivationGoal::*GoalState)(); GoalState state; - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - /* The final output paths of the build. - For input-addressed derivations, always the precomputed paths @@ -191,11 +116,6 @@ struct DerivationGoal : public Goal BuildMode buildMode; - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - BuildResult result; /* The current round, if we're building multiple times. */ @@ -203,17 +123,6 @@ struct DerivationGoal : public Goal size_t nrRounds; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; @@ -226,39 +135,13 @@ struct DerivationGoal : public Goal /* The remote machine on which we're building. */ std::string machineName; - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); + virtual ~DerivationGoal(); void timedOut(Error && ex) override; @@ -280,7 +163,7 @@ struct DerivationGoal : public Goal void closureRepaired(); void inputsRealised(); void tryToBuild(); - void tryLocalBuild(); + virtual void tryLocalBuild(); void buildDone(); void resolvedFinished(); @@ -288,40 +171,11 @@ struct DerivationGoal : public Goal /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); + virtual int getChildStatus(); /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); + virtual void registerOutputs(); /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -329,8 +183,18 @@ struct DerivationGoal : public Goal /* Close the log file. */ void closeLogFile(); - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); + /* Close the read side of the logger pipe. */ + virtual void closeReadPipes(); + + /* Cleanup hooks for buildDone() */ + virtual void cleanupHookFinally(); + virtual void cleanupPreChildKill(); + virtual void cleanupPostChildKill(); + virtual bool cleanupDecideWhetherDiskFull(); + virtual void cleanupPostOutputsRegisteredModeCheck(); + virtual void cleanupPostOutputsRegisteredModeNonCheck(); + + virtual bool isReadDesc(int fd); /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; @@ -347,17 +211,7 @@ struct DerivationGoal : public Goal void checkPathValidity(); /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); + virtual void killChild(); void repairClosure(); @@ -370,4 +224,6 @@ struct DerivationGoal : public Goal StorePathSet exportReferences(const StorePathSet & storePaths); }; +MakeError(NotDeterministic, BuildError); + } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 3a05a022c..01a564aba 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,6 +2,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" +#include "local-store.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 924c69fb7..23ffe740a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,4 @@ -#include "derivation-goal.hh" +#include "local-derivation-goal.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -75,7 +75,6 @@ void handleDiffHook( diffHookOptions.uid = uid; diffHookOptions.gid = gid; diffHookOptions.chdir = "/"; - auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) throw ExecError(diffRes.first, @@ -94,8 +93,9 @@ void handleDiffHook( } } -const Path DerivationGoal::homeDir = "/homeless-shelter"; +const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +#if 0 DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -138,19 +138,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation garbage-collected. (See isActiveTempFile() in gc.cc.) */ worker.store.addTempRoot(this->drvPath); } +#endif -DerivationGoal::~DerivationGoal() +LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ + try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { killChild(); } catch (...) { ignoreException(); } try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } } +#if 0 string DerivationGoal::key() { /* Ensure that derivations get built in order of their name, @@ -159,9 +160,10 @@ string DerivationGoal::key() derivation goals (due to "b$"). */ return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } +#endif -inline bool DerivationGoal::needsHashRewrite() +inline bool LocalDerivationGoal::needsHashRewrite() { #if __linux__ return !useChroot; @@ -172,7 +174,15 @@ inline bool DerivationGoal::needsHashRewrite() } -void DerivationGoal::killChild() +LocalStore & LocalDerivationGoal::getLocalStore() +{ + auto p = dynamic_cast(&worker.store); + assert(p); + return *p; +} + + +void LocalDerivationGoal::killChild() { if (pid != -1) { worker.childTerminated(this); @@ -193,17 +203,11 @@ void DerivationGoal::killChild() assert(pid == -1); } - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); + DerivationGoal::killChild(); } +#if 0 void DerivationGoal::work() { (this->*state)(); @@ -695,15 +699,9 @@ void DerivationGoal::tryToBuild() state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); } +#endif -void DerivationGoal::tryLocalBuild() { - /* Make sure that we are allowed to start a build. */ - if (!dynamic_cast(&worker.store)) { - throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - } +void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); @@ -757,7 +755,6 @@ void DerivationGoal::tryLocalBuild() { started(); } - static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) @@ -785,51 +782,125 @@ static void movePath(const Path & src, const Path & dst) } -void replaceValidPath(const Path & storePath, const Path & tmpPath) +extern void replaceValidPath(const Path & storePath, const Path & tmpPath); + + +int LocalDerivationGoal::getChildStatus() { - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); + return hook ? DerivationGoal::getChildStatus() : pid.kill(); +} - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); +void LocalDerivationGoal::closeReadPipes() +{ + if (hook) { + DerivationGoal::closeReadPipes(); + } else + builderOut.readSide = -1; } -MakeError(NotDeterministic, BuildError); +void LocalDerivationGoal::cleanupHookFinally() +{ + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + buildUser.reset(); +} +void LocalDerivationGoal::cleanupPreChildKill() +{ + sandboxMountNamespace = -1; +} + + +void LocalDerivationGoal::cleanupPostChildKill() +{ + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); +} + + +bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() +{ + bool diskFull = false; + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + { + auto & localStore = getLocalStore(); + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } + + return diskFull; +} + + +void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck() +{ + deleteTmpDir(true); +} + + +void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() +{ + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(worker.store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + cleanupPostOutputsRegisteredModeCheck(); +} + + +#if 0 void DerivationGoal::buildDone() { trace("build done"); - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); + Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - sandboxMountNamespace = -1; + cleanupPreChildKill(); /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); + int status = getChildStatus(); debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); @@ -840,24 +911,12 @@ void DerivationGoal::buildDone() worker.childTerminated(this); /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; + closeReadPipes(); /* Close the log file. */ closeLogFile(); - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); + cleanupPostChildKill(); bool diskFull = false; @@ -866,36 +925,7 @@ void DerivationGoal::buildDone() /* Check the exit status. */ if (!statusOk(status)) { - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - if (auto localStore = dynamic_cast(&worker.store)) { - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } + diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", yellowtxt(worker.store.printStorePath(drvPath)), @@ -975,19 +1005,12 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeCheck(); done(BuildResult::Built); return; } - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { @@ -1169,15 +1192,17 @@ HookReply DerivationGoal::tryBuildHook() return rpAccept; } +#endif int childEntry(void * arg) { - ((DerivationGoal *) arg)->runChild(); + ((LocalDerivationGoal *) arg)->runChild(); return 1; } +#if 0 StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { StorePathSet paths; @@ -1212,6 +1237,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) return paths; } +#endif static std::once_flag dns_resolve_flag; @@ -1230,7 +1256,7 @@ static void preloadNSS() { } -void linkOrCopy(const Path & from, const Path & to) +static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a @@ -1247,7 +1273,7 @@ void linkOrCopy(const Path & from, const Path & to) } -void DerivationGoal::startBuilder() +void LocalDerivationGoal::startBuilder() { /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) @@ -1285,15 +1311,13 @@ void DerivationGoal::startBuilder() useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } - if (auto localStoreP = dynamic_cast(&worker.store)) { - auto & localStore = *localStoreP; - if (localStore.storeDir != localStore.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif } /* Create a temporary directory where the build will take @@ -1850,7 +1874,7 @@ void DerivationGoal::startBuilder() } -void DerivationGoal::initTmpDir() { +void LocalDerivationGoal::initTmpDir() { /* In a sandbox, for determinism, always use the same temporary directory. */ #if __linux__ @@ -1899,7 +1923,7 @@ void DerivationGoal::initTmpDir() { } -void DerivationGoal::initEnv() +void LocalDerivationGoal::initEnv() { env.clear(); @@ -1960,7 +1984,7 @@ void DerivationGoal::initEnv() static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); -void DerivationGoal::writeStructuredAttrs() +void LocalDerivationGoal::writeStructuredAttrs() { auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; @@ -2079,9 +2103,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo { ref next; - DerivationGoal & goal; + LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) : StoreConfig(params) , LocalFSStoreConfig(params) , RestrictedStoreConfig(params) @@ -2256,15 +2280,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo }; -void DerivationGoal::startDaemon() +void LocalDerivationGoal::startDaemon() { settings.requireExperimentalFeature("recursive-nix"); Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - if (auto localStore = dynamic_cast(&worker.store)) - params["root"] = localStore->rootDir; + params["root"] = getLocalStore().rootDir; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -2322,7 +2345,7 @@ void DerivationGoal::startDaemon() } -void DerivationGoal::stopDaemon() +void LocalDerivationGoal::stopDaemon() { if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) throw SysError("shutting down daemon socket"); @@ -2340,7 +2363,7 @@ void DerivationGoal::stopDaemon() } -void DerivationGoal::addDependency(const StorePath & path) +void LocalDerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; @@ -2397,8 +2420,7 @@ void DerivationGoal::addDependency(const StorePath & path) } } - -void DerivationGoal::chownToBuilder(const Path & path) +void LocalDerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) @@ -2469,7 +2491,7 @@ void setupSeccomp() } -void DerivationGoal::runChild() +void LocalDerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -2977,7 +2999,7 @@ void DerivationGoal::runChild() } -void DerivationGoal::registerOutputs() +void LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2987,14 +3009,8 @@ void DerivationGoal::registerOutputs() floating content-addressed derivations this isn't the case. */ if (hook) { - bool allValid = true; - for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!outputPath || !worker.store.isValidPath(*outputPath)) - allValid = false; - else - finalOutputs.insert_or_assign(outputName, *outputPath); - } - if (allValid) return; + DerivationGoal::registerOutputs(); + return; } std::map infos; @@ -3349,10 +3365,7 @@ void DerivationGoal::registerOutputs() } } - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; + auto & localStore = getLocalStore(); if (buildMode == bmCheck) { @@ -3481,10 +3494,7 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ { - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; + auto & localStore = getLocalStore(); ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { @@ -3513,7 +3523,7 @@ void DerivationGoal::registerOutputs() } -void DerivationGoal::checkOutputs(const std::map & outputs) +void LocalDerivationGoal::checkOutputs(const std::map & outputs) { std::map outputsByPath; for (auto & output : outputs) @@ -3678,6 +3688,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) } +#if 0 Path DerivationGoal::openLogFile() { logSize = 0; @@ -3720,9 +3731,10 @@ void DerivationGoal::closeLogFile() logSink = logFileSink = 0; fdLogFile = -1; } +#endif -void DerivationGoal::deleteTmpDir(bool force) +void LocalDerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { /* Don't keep temporary directories for builtins because they @@ -3738,10 +3750,17 @@ void DerivationGoal::deleteTmpDir(bool force) } +bool LocalDerivationGoal::isReadDesc(int fd) +{ + return (hook && DerivationGoal::isReadDesc(fd)) || + (!hook && fd == builderOut.readSide.get()); +} + + +#if 0 void DerivationGoal::handleChildOutput(int fd, const string & data) { - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) + if (isReadDesc(fd)) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { @@ -3855,9 +3874,10 @@ void DerivationGoal::checkPathValidity() } } } +#endif -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), @@ -3865,7 +3885,7 @@ StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) } -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) +StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), @@ -3873,6 +3893,7 @@ StorePath DerivationGoal::makeFallbackPath(const StorePath & path) } +#if 0 void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; @@ -3897,6 +3918,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional ex) worker.updateProgress(); } +#endif } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 6dc164922..f7994113e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,48 +1,15 @@ #pragma once -#include "parsed-derivations.hh" -#include "lock.hh" +#include "derivation-goal.hh" #include "local-store.hh" -#include "goal.hh" namespace nix { -using std::map; - -struct HookInstance; - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - Hash outputHash; - std::optional known; -}; - -struct DerivationGoal : public Goal +struct LocalDerivationGoal : public DerivationGoal { + LocalStore & getLocalStore(); + +#if 0 /* Whether to use an on-disk .drv file. */ bool useDerivation; @@ -78,6 +45,7 @@ struct DerivationGoal : public Goal StorePathSet inputPaths; std::map initialOutputs; +#endif /* User selected for running the builder. */ std::unique_ptr buildUser; @@ -91,6 +59,7 @@ struct DerivationGoal : public Goal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; +#if 0 /* File descriptor for the log file. */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; @@ -105,6 +74,7 @@ struct DerivationGoal : public Goal size_t currentLogLinePos = 0; // to handle carriage return std::string currentHookLine; +#endif /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -120,8 +90,10 @@ struct DerivationGoal : public Goal namespace. */ bool usingUserNamespace = true; +#if 0 /* The build hook. */ std::unique_ptr hook; +#endif /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -131,14 +103,18 @@ struct DerivationGoal : public Goal /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; +#if 0 /* The sort of derivation we are building. */ DerivationType derivationType; +#endif /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; +#if 0 typedef void (DerivationGoal::*GoalState)(); GoalState state; +#endif /* Stuff we need to pass to initChild(). */ struct ChrootPath { @@ -179,6 +155,7 @@ struct DerivationGoal : public Goal */ OutputPathMap scratchOutputs; +#if 0 /* The final output paths of the build. - For input-addressed derivations, always the precomputed paths @@ -190,18 +167,21 @@ struct DerivationGoal : public Goal OutputPathMap finalOutputs; BuildMode buildMode; +#endif /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ StorePathSet redirectedBadOutputs; +#if 0 BuildResult result; /* The current round, if we're building multiple times. */ size_t curRound = 1; size_t nrRounds; +#endif /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it @@ -214,6 +194,7 @@ struct DerivationGoal : public Goal const static Path homeDir; +#if 0 std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; @@ -225,6 +206,7 @@ struct DerivationGoal : public Goal /* The remote machine on which we're building. */ std::string machineName; +#endif /* The recursive Nix daemon socket. */ AutoCloseFD daemonSocket; @@ -249,17 +231,14 @@ struct DerivationGoal : public Goal friend struct RestrictedStore; - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); + using DerivationGoal::DerivationGoal; + + virtual ~LocalDerivationGoal() override; /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); +#if 0 void timedOut(Error && ex) override; string key() override; @@ -280,13 +259,16 @@ struct DerivationGoal : public Goal void closureRepaired(); void inputsRealised(); void tryToBuild(); - void tryLocalBuild(); +#endif + void tryLocalBuild() override; +#if 0 void buildDone(); void resolvedFinished(); /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); +#endif /* Start building a derivation. */ void startBuilder(); @@ -311,27 +293,46 @@ struct DerivationGoal : public Goal /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); + int getChildStatus() override; + /* Run the builder's process. */ void runChild(); /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs(); + void registerOutputs() override; /* Check that an output meets the requirements specified by the 'outputChecks' attribute (or the legacy '{allowed,disallowed}{References,Requisites}' attributes). */ void checkOutputs(const std::map & outputs); +#if 0 /* Open a log file and a pipe to it. */ Path openLogFile(); /* Close the log file. */ void closeLogFile(); +#endif + + /* Close the read side of the logger pipe. */ + void closeReadPipes() override; + + /* Cleanup hooks for buildDone() */ + void cleanupHookFinally() override; + void cleanupPreChildKill() override; + void cleanupPostChildKill() override; + bool cleanupDecideWhetherDiskFull() override; + void cleanupPostOutputsRegisteredModeCheck() override; + void cleanupPostOutputsRegisteredModeNonCheck() override; + + bool isReadDesc(int fd) override; + /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); +#if 0 /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; @@ -345,9 +346,10 @@ struct DerivationGoal : public Goal /* Return the set of (in)valid paths. */ void checkPathValidity(); +#endif /* Forcibly kill the child process, if any. */ - void killChild(); + void killChild() override; /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) @@ -359,6 +361,7 @@ struct DerivationGoal : public Goal rewrites caught everything */ StorePath makeFallbackPath(std::string_view outputName); +#if 0 void repairClosure(); void started(); @@ -368,6 +371,7 @@ struct DerivationGoal : public Goal std::optional ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); +#endif }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2f13aa885..b2223c3b6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,7 +1,7 @@ #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" -#include "derivation-goal.hh" +#include "local-derivation-goal.hh" #include "hook-instance.hh" #include @@ -59,8 +59,10 @@ std::shared_ptr Worker::makeDerivationGoalCommon( std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) + : std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } @@ -68,8 +70,10 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) + : std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 780cc0f07..03bb0218d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -280,7 +280,7 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend struct DerivationGoal; + friend struct LocalDerivationGoal; friend struct SubstitutionGoal; }; From d560311f7643096ce815a7c655a077621abb7d1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 15:31:15 +0000 Subject: [PATCH 409/477] Remove temporary `#if 0...#endif` from previous commit --- src/libstore/build/local-derivation-goal.cc | 1090 ------------------- src/libstore/build/local-derivation-goal.hh | 175 +-- 2 files changed, 1 insertion(+), 1264 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 23ffe740a..3a0616864 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -95,51 +95,6 @@ void handleDiffHook( const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; -#if 0 -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(drv); - - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} -#endif - LocalDerivationGoal::~LocalDerivationGoal() { @@ -151,18 +106,6 @@ LocalDerivationGoal::~LocalDerivationGoal() } -#if 0 -string DerivationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); -} -#endif - - inline bool LocalDerivationGoal::needsHashRewrite() { #if __linux__ @@ -207,500 +150,6 @@ void LocalDerivationGoal::killChild() } -#if 0 -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::make_unique(worker.store.derivationFromPath(drvPath)); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - auto outputHashes = staticOutputHashes(worker.store, *drv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(upcast_goal(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. - - If the nrIncompleteClosure != nrFailed, we have another issue as well. - In particular, it may be the case that the hole in the closure is - an output of the current derivation, which causes a loop if retried. - */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - printError( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (settings.isExperimentalFeatureEnabled("ca-derivations") && - ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed)) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - resolvedDrv = drvResolved; - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - if (dynamic_cast(&worker.store)) - /* If we aren't a local store, we might need to use the local store as - a build remote, but that would cause a deadlock. */ - /* FIXME: Make it so we can use ourselves as a build remote even if we - are the local store (separate locking for building vs scheduling? */ - /* FIXME: find some way to lock for scheduling for the other stores so - a forking daemon with --store still won't farm out redundant builds. - */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} -#endif - void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { @@ -887,314 +336,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -#if 0 -void DerivationGoal::buildDone() -{ - trace("build done"); - - Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - - cleanupPreChildKill(); - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = getChildStatus(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - closeReadPipes(); - - /* Close the log file. */ - closeLogFile(); - - cleanupPostChildKill(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= cleanupDecideWhetherDiskFull(); - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += fmt(";\nlast %d log lines:\n", logTail.size()); - for (auto & line : logTail) { - msg += "> "; - msg += line; - msg += "\n"; - } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", - worker.store.printStorePath(drvPath)); - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (std::string_view data) override { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built); - return; - } - - cleanupPostOutputsRegisteredModeNonCheck(); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - assert(resolvedDrv); - - auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); - - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv->outputNames(); - - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} - ); - // We've just built it, but maybe the build failed, in which case the - // realisation won't be there - if (realisation) { - auto newRealisation = *realisation; - newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; - worker.store.registerDrvOutput(newRealisation); - } else { - // If we don't have a realisation, then it must mean that something - // failed when building the resolved drv - assert(!result.success()); - } - } - - // This is potentially a bit fishy in terms of error reporting. Not sure - // how to do it in a cleaner way - amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - auto s = [&]() { - try { - return readLine(worker.hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the response from the build hook"); - throw e; - } - }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - printError( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - try { - machineName = readLine(hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the machine name from the build hook"); - throw e; - } - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StringSet missingOutputs; - for (auto & [outputName, status] : initialOutputs) { - // XXX: Does this include known CA outputs? - if (buildMode != bmCheck && status.known && status.known->isValid()) continue; - missingOutputs.insert(outputName); - } - worker_proto::write(worker.store, hook->sink, missingOutputs); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} -#endif - - int childEntry(void * arg) { ((LocalDerivationGoal *) arg)->runChild(); @@ -1202,43 +343,6 @@ int childEntry(void * arg) } -#if 0 -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} -#endif - static std::once_flag dns_resolve_flag; static void preloadNSS() { @@ -3688,52 +2792,6 @@ void LocalDerivationGoal::checkOutputs(const std::map & out } -#if 0 -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path logDir; - if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; - else - logDir = settings.nixLogDir; - Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} -#endif - - void LocalDerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { @@ -3757,126 +2815,6 @@ bool LocalDerivationGoal::isReadDesc(int fd) } -#if 0 -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if (isReadDesc(fd)) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput & info = initialOutputs.at(i.first); - info.wanted = wantOutput(i.first, wantedOutputs); - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (auto real = worker.store.queryRealisation( - DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { - info.known = { - .path = real->outPath, - .status = PathStatus::Valid, - }; - } - } - } -} -#endif - - StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) { return worker.store.makeStorePath( @@ -3893,32 +2831,4 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) } -#if 0 -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} -#endif - - } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f7994113e..a2b386a72 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -9,44 +9,6 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); -#if 0 - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The path of the corresponding resolved derivation */ - std::optional resolvedDrv; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; -#endif - /* User selected for running the builder. */ std::unique_ptr buildUser; @@ -59,23 +21,6 @@ struct LocalDerivationGoal : public DerivationGoal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; -#if 0 - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; -#endif - /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -90,11 +35,6 @@ struct LocalDerivationGoal : public DerivationGoal namespace. */ bool usingUserNamespace = true; -#if 0 - /* The build hook. */ - std::unique_ptr hook; -#endif - /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -103,19 +43,9 @@ struct LocalDerivationGoal : public DerivationGoal /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; -#if 0 - /* The sort of derivation we are building. */ - DerivationType derivationType; -#endif - /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; -#if 0 - typedef void (DerivationGoal::*GoalState)(); - GoalState state; -#endif - /* Stuff we need to pass to initChild(). */ struct ChrootPath { Path source; @@ -155,34 +85,11 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; -#if 0 - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; -#endif - /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ StorePathSet redirectedBadOutputs; -#if 0 - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; -#endif - /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same @@ -194,20 +101,6 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; -#if 0 - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; -#endif - /* The recursive Nix daemon socket. */ AutoCloseFD daemonSocket; @@ -238,37 +131,8 @@ struct LocalDerivationGoal : public DerivationGoal /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); -#if 0 - void timedOut(Error && ex) override; - - string key() override; - - void work() override; - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); -#endif + /* The additional states. */ void tryLocalBuild() override; -#if 0 - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); -#endif /* Start building a derivation. */ void startBuilder(); @@ -307,14 +171,6 @@ struct LocalDerivationGoal : public DerivationGoal '{allowed,disallowed}{References,Requisites}' attributes). */ void checkOutputs(const std::map & outputs); -#if 0 - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); -#endif - /* Close the read side of the logger pipe. */ void closeReadPipes() override; @@ -328,26 +184,9 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); -#if 0 - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); -#endif - /* Forcibly kill the child process, if any. */ void killChild() override; @@ -360,18 +199,6 @@ struct LocalDerivationGoal : public DerivationGoal /* FIXME add option to randomize, so we can audit whether our rewrites caught everything */ StorePath makeFallbackPath(std::string_view outputName); - -#if 0 - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -#endif }; } From 553b79f8c980fde70fe186ee4980b2d12e27d756 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Feb 2021 17:38:38 +0000 Subject: [PATCH 410/477] Remove unused `redirectedBadOutputs` --- src/libstore/build/local-derivation-goal.cc | 4 ---- src/libstore/build/local-derivation-goal.hh | 5 ----- 2 files changed, 9 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3a0616864..9c2f1dda6 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -460,10 +460,6 @@ void LocalDerivationGoal::startBuilder() makeFallbackPath(status.known->path); scratchOutputs.insert_or_assign(outputName, scratchPath); - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - /* Substitute output placeholders with the scratch output paths. We'll use during the build. */ inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index a2b386a72..4bbf27a1b 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -85,11 +85,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same From 5b42e5b1771061de50575b33eeeda56f40f216f2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 16:29:19 +0000 Subject: [PATCH 411/477] Restore now-working build-remote-content-addressed-fixed test This was - Added in dbf96e10ecc75410c9db798f208f8a8310842a4f. - Commented out in 07975979aae4e7729ae13ffeb7390d07d71ad4bd, which I believe only reached master by mistake. - Deleted in c32168c9bc161e0c9cea027853895971699510cb, when `tests/build-hook-ca.nix` was reused for a new test. But the test works, and we ought to have it. --- tests/build-hook-ca-fixed.nix | 56 +++++++++++++++++++ ...hook-ca.nix => build-hook-ca-floating.nix} | 0 tests/build-remote-content-addressed-fixed.sh | 5 ++ ...build-remote-content-addressed-floating.sh | 2 +- tests/local.mk | 1 + 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/build-hook-ca-fixed.nix rename tests/{build-hook-ca.nix => build-hook-ca-floating.nix} (100%) create mode 100644 tests/build-remote-content-addressed-fixed.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix new file mode 100644 index 000000000..ec7171ac9 --- /dev/null +++ b/tests/build-hook-ca-fixed.nix @@ -0,0 +1,56 @@ +{ busybox }: + +with import ./config.nix; + +let + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo FOO > $out"; + requiredSystemFeatures = ["foo"]; + outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; + }; + + input2 = mkDerivation { + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo BAR > $out"; + requiredSystemFeatures = ["bar"]; + outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + read x < ${input2} + echo $x BAZ > $out + ''; + requiredSystemFeatures = ["baz"]; + outputHash = "sha256-daKAcPp/+BYMQsVi/YYMlCKoNAxCNDsaivwSHgQqD2s="; + }; + +in + + mkDerivation { + shell = busybox; + name = "build-remote"; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + outputHash = "sha256-5SxbkUw6xe2l9TE1uwCvTtTDysD1vhRor38OtDF0LqQ="; + } diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca.nix rename to tests/build-hook-ca-floating.nix diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh new file mode 100644 index 000000000..ae7441591 --- /dev/null +++ b/tests/build-remote-content-addressed-fixed.sh @@ -0,0 +1,5 @@ +source common.sh + +file=build-hook-ca-fixed.nix + +source build-remote.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index cbb75729b..7447d92bd 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -1,6 +1,6 @@ source common.sh -file=build-hook-ca.nix +file=build-hook-ca-floating.nix sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf diff --git a/tests/local.mk b/tests/local.mk index 7deea9ac1..4d970d5e4 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ ssh-relay.sh \ nar-access.sh \ From 12ec962dd8a6d8058ba11e517d74f6a07b3dc903 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 25 Feb 2021 16:12:51 -0600 Subject: [PATCH 412/477] simplify changing cachix cache for install tests - convert cachix cache name from an env into a secret so it (along with the token/key) can be set once per fork - use CACHIX_AUTH_TOKEN in addition to CACHIX_SIGNING_KEY; it looks like cachix will try signing key first, then auth token. --- .github/workflows/test.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bde6106e0..2531a7d35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,52 +8,62 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci + steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v12 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v8 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) - installer: - if: github.event_name == 'push' - needs: tests + check_cachix: + name: Cachix secret present for installer tests + runs-on: ubuntu-latest + outputs: + secret: ${{ steps.secret.outputs.secret }} + steps: + - name: Check for Cachix secret + id: secret + env: + _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} + run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + installer: + needs: [tests, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' runs-on: ubuntu-latest - env: - CACHIX_NAME: nix-ci outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v12 - uses: cachix/cachix-action@v8 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - id: prepare-installer run: scripts/prepare-installer-for-github-actions installer_test: - if: github.event_name == 'push' - needs: installer + needs: [installer, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci steps: - uses: actions/checkout@v2.3.4 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@master with: install_url: '${{needs.installer.outputs.installerURL}}' - install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve' + install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - run: nix-instantiate -E 'builtins.currentTime' --eval - \ No newline at end of file From bd0b0f9ab7655553f64f158d5d9a9445f5604abd Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 21:48:41 +0000 Subject: [PATCH 413/477] mk: add support for CPPFLAGS --- mk/lib.mk | 1 + mk/patterns.mk | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mk/lib.mk b/mk/lib.mk index a09ebaa97..6b92136cd 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -153,4 +153,5 @@ endif @echo " CFLAGS: Flags for the C compiler" @echo " CXX ($(CXX)): C++ compiler to be used" @echo " CXXFLAGS: Flags for the C++ compiler" + @echo " CPPFLAGS: C preprocessor flags, used for both CC and CXX" @$(print-var-help) diff --git a/mk/patterns.mk b/mk/patterns.mk index 7319f4cdd..86a724806 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,11 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP From 7241fdc3d2386d256ca8870ca955b498d0ac2ff7 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 22:06:06 +0000 Subject: [PATCH 414/477] Properly propagate libseccomp linker flags --- Makefile.config.in | 1 + src/libstore/local.mk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index 9d0500e48..3c1f01d1e 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -17,6 +17,7 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ +LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SHELL = @bash@ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 03c4351ac..cf0933705 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -28,7 +28,7 @@ ifeq ($(OS), SunOS) endif ifeq ($(HAVE_SECCOMP), 1) - libstore_LDFLAGS += -lseccomp + libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif libstore_CXXFLAGS += \ From 2d7917f035c7396e87546b130317a2e5234afa36 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 21:42:51 +0000 Subject: [PATCH 415/477] Revert "Add support for building JARs from Java sources" This reverts commit 259086de841d155f7951c2cc50f799a4631aa512. --- mk/jars.mk | 36 ------------------------------------ mk/lib.mk | 12 +----------- mk/tracing.mk | 2 -- 3 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 mk/jars.mk diff --git a/mk/jars.mk b/mk/jars.mk deleted file mode 100644 index c8513e664..000000000 --- a/mk/jars.mk +++ /dev/null @@ -1,36 +0,0 @@ -define build-jar - - $(1)_NAME ?= $(1) - - _d := $$(strip $$($(1)_DIR)) - - $(1)_PATH := $$(_d)/$$($(1)_NAME).jar - - $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp - - _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH)) - - $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER) - @rm -rf $$($(1)_TMPDIR) - @mkdir -p $$($(1)_TMPDIR) - $$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \ - $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \ - -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH" - @echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest - $$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) . - @rm $$($(1)_PATH).manifest - @rm -rf $$($(1)_TMPDIR) - - $(1)_INSTALL_DIR ?= $$(jardir) - - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar - - $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644)) - - install: $$($(1)_INSTALL_PATH) - - jars-list += $$($(1)_PATH) - - clean-files += $$($(1)_PATH) - -endef diff --git a/mk/lib.mk b/mk/lib.mk index a09ebaa97..6a1c465b6 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -31,7 +31,6 @@ libdir ?= $(prefix)/lib bindir ?= $(prefix)/bin libexecdir ?= $(prefix)/libexec datadir ?= $(prefix)/share -jardir ?= $(datadir)/java localstatedir ?= $(prefix)/var sysconfdir ?= $(prefix)/etc mandir ?= $(prefix)/share/man @@ -74,7 +73,6 @@ BUILD_DEBUG ?= 1 ifeq ($(BUILD_DEBUG), 1) GLOBAL_CFLAGS += -g GLOBAL_CXXFLAGS += -g - GLOBAL_JAVACFLAGS += -g endif @@ -84,7 +82,6 @@ include mk/clean.mk include mk/install.mk include mk/libraries.mk include mk/programs.mk -include mk/jars.mk include mk/patterns.mk include mk/templates.mk include mk/tests.mk @@ -102,7 +99,6 @@ $(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) # Instantiate stuff. $(foreach lib, $(libraries), $(eval $(call build-library,$(lib)))) $(foreach prog, $(programs), $(eval $(call build-program,$(prog)))) -$(foreach jar, $(jars), $(eval $(call build-jar,$(jar)))) $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir)))) $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) @@ -113,7 +109,7 @@ $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/ .PHONY: default all man help -all: $(programs-list) $(libs-list) $(jars-list) $(man-pages) +all: $(programs-list) $(libs-list) $(man-pages) man: $(man-pages) @@ -137,12 +133,6 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done -endif -ifdef jars-list - @echo "" - @echo "The following JARs can be built:" - @echo "" - @for i in $(jars-list); do echo " $$i"; done endif @echo "" @echo "The following variables control the build:" diff --git a/mk/tracing.mk b/mk/tracing.mk index 54c77ab60..1fc5573d7 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -8,8 +8,6 @@ ifeq ($(V), 0) trace-ld = @echo " LD " $@; trace-ar = @echo " AR " $@; trace-install = @echo " INST " $@; - trace-javac = @echo " JAVAC " $@; - trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; From ae1441e5488a0e1608851b329358eb390a08ac27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 27 Feb 2021 05:23:14 +0000 Subject: [PATCH 416/477] Fix testing fixed-output derivations in double sandboxes What happened was that Nix was trying to unconditionally mount these paths in fixed-output derivations, but since the outer derivation was pure, those paths did not exist. The solution is to only mount those paths when they exist. --- src/libstore/build/local-derivation-goal.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c2f1dda6..90731d98d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -287,7 +287,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - { + { auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; @@ -297,7 +297,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (statvfs(tmpDir.c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; - } + } #endif deleteTmpDir(false); @@ -1703,18 +1703,18 @@ void LocalDerivationGoal::runChild() network, so give them access to /etc/resolv.conf and so on. */ if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + if (pathExists(path)) + ss.push_back(path); } for (auto & i : ss) dirsInChroot.emplace(i, i); From 4bbd80c5366711b8f1b5ad108ba22206d3bee783 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Feb 2021 21:50:50 +0000 Subject: [PATCH 417/477] Throw error for derivation goal with bogus wanted output --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c29237f5c..530f8829a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1243,9 +1243,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; + auto wantedOutputsLeft = wantedOutputs; for (auto & i : queryPartialDerivationOutputMap()) { InitialOutput & info = initialOutputs.at(i.first); info.wanted = wantOutput(i.first, wantedOutputs); + if (info.wanted) + wantedOutputsLeft.erase(i.first); if (i.second) { auto outputPath = *i.second; info.known = { @@ -1267,6 +1270,11 @@ void DerivationGoal::checkPathValidity() } } } + // If we requested all the outputs via the empty set, we are always fine. + // If we requested specific elements, the loop above removes all the valid + // ones, so any that are left must be invalid. + if (!wantedOutputsLeft.empty()) + throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft)); } From 259d6778efd865ccd3b5fbf4f3a29002a7d58d93 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 16:04:18 +0100 Subject: [PATCH 418/477] Move the CA tests to a sub-directory Requires a slight update to the test infra to work properly, but having the possibility to group tests that way makes the whole thing quite cleaner imho --- mk/run_test.sh | 2 +- mk/tests.mk | 2 +- tests/{content-addressed.sh => ca/build.sh} | 2 ++ tests/ca/common.sh | 1 + tests/{ => ca}/content-addressed.nix | 2 +- tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} | 0 tests/common.sh.in | 2 +- tests/local.mk | 6 +++--- 8 files changed, 10 insertions(+), 7 deletions(-) rename tests/{content-addressed.sh => ca/build.sh} (98%) create mode 100644 tests/ca/common.sh rename tests/{ => ca}/content-addressed.nix (98%) rename tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} (100%) diff --git a/mk/run_test.sh b/mk/run_test.sh index 6af5b070a..3783d3bf7 100755 --- a/mk/run_test.sh +++ b/mk/run_test.sh @@ -14,7 +14,7 @@ if [ -t 1 ]; then yellow="" normal="" fi -(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) +(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" status=$? if [ $status -eq 0 ]; then diff --git a/mk/tests.mk b/mk/tests.mk index c1e140bac..21bdc5748 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null + @env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null endef diff --git a/tests/content-addressed.sh b/tests/ca/build.sh similarity index 98% rename from tests/content-addressed.sh rename to tests/ca/build.sh index 7e32e1f28..35bf1dcf7 100644 --- a/tests/content-addressed.sh +++ b/tests/ca/build.sh @@ -61,7 +61,9 @@ testNixCommand () { # Disabled until we have it properly working # testRemoteCache +clearStore testDeterministicCA +clearStore testCutoff testGC testNixCommand diff --git a/tests/ca/common.sh b/tests/ca/common.sh new file mode 100644 index 000000000..e083d873c --- /dev/null +++ b/tests/ca/common.sh @@ -0,0 +1 @@ +source ../common.sh diff --git a/tests/content-addressed.nix b/tests/ca/content-addressed.nix similarity index 98% rename from tests/content-addressed.nix rename to tests/ca/content-addressed.nix index 61079176f..e5b1c4de3 100644 --- a/tests/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ../config.nix; { seed ? 0 }: # A simple content-addressed derivation. diff --git a/tests/nix-copy-content-addressed.sh b/tests/ca/nix-copy.sh similarity index 100% rename from tests/nix-copy-content-addressed.sh rename to tests/ca/nix-copy.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index e3bcab507..de44a4da4 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,7 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket +export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then diff --git a/tests/local.mk b/tests/local.mk index 7deea9ac1..07cfd7a50 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,10 +38,10 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ - content-addressed.sh \ - nix-copy-content-addressed.sh \ build.sh \ - compute-levels.sh + compute-levels.sh \ + ca/build.sh \ + ca/nix-copy.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 5d1c05b07561c841c68eb3ff9698ce9d2355fe41 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 13:47:06 +0100 Subject: [PATCH 419/477] SubstitutionGoal -> PathSubstitutionGoal To prepare for the upcoming DrvOutputSubstitutionGoal --- src/libstore/build/derivation-goal.cc | 8 +++---- src/libstore/build/entry-points.cc | 8 +++---- src/libstore/build/substitution-goal.cc | 32 ++++++++++++------------- src/libstore/build/substitution-goal.hh | 9 +++---- src/libstore/build/worker.cc | 12 +++++----- src/libstore/build/worker.hh | 12 +++++----- src/libstore/local-store.hh | 2 ++ 7 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c29237f5c..7b97e575a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -170,7 +170,7 @@ void DerivationGoal::getDerivation() return; } - addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); state = &DerivationGoal::loadDerivation; } @@ -253,7 +253,7 @@ void DerivationGoal::haveDerivation() /* Nothing to wait for; tail call */ return DerivationGoal::gaveUpOnSubstitution(); } - addWaitee(upcast_goal(worker.makeSubstitutionGoal( + addWaitee(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)))); @@ -337,7 +337,7 @@ void DerivationGoal::gaveUpOnSubstitution() if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -388,7 +388,7 @@ void DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 01a564aba..686364440 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector & drvPaths, Build if (path.path.isDerivation()) goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); @@ -31,7 +31,7 @@ void Store::buildPaths(const std::vector & drvPaths, Build } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -90,7 +90,7 @@ void Store::ensurePath(const StorePath & path) if (isValidPath(path)) return; Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; worker.run(goals); @@ -108,7 +108,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index c4b0de78d..5d88b8758 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -5,20 +5,20 @@ namespace nix { -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) , ca(ca) { - state = &SubstitutionGoal::init; + state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); } -SubstitutionGoal::~SubstitutionGoal() +PathSubstitutionGoal::~PathSubstitutionGoal() { try { if (thr.joinable()) { @@ -32,13 +32,13 @@ SubstitutionGoal::~SubstitutionGoal() } -void SubstitutionGoal::work() +void PathSubstitutionGoal::work() { (this->*state)(); } -void SubstitutionGoal::init() +void PathSubstitutionGoal::init() { trace("init"); @@ -59,7 +59,7 @@ void SubstitutionGoal::init() } -void SubstitutionGoal::tryNext() +void PathSubstitutionGoal::tryNext() { trace("trying next substituter"); @@ -154,16 +154,16 @@ void SubstitutionGoal::tryNext() paths referenced by this one. */ for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); else - state = &SubstitutionGoal::referencesValid; + state = &PathSubstitutionGoal::referencesValid; } -void SubstitutionGoal::referencesValid() +void PathSubstitutionGoal::referencesValid() { trace("all references realised"); @@ -177,12 +177,12 @@ void SubstitutionGoal::referencesValid() if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &SubstitutionGoal::tryToRun; + state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); } -void SubstitutionGoal::tryToRun() +void PathSubstitutionGoal::tryToRun() { trace("trying to run"); @@ -221,11 +221,11 @@ void SubstitutionGoal::tryToRun() worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - state = &SubstitutionGoal::finished; + state = &PathSubstitutionGoal::finished; } -void SubstitutionGoal::finished() +void PathSubstitutionGoal::finished() { trace("substitute finished"); @@ -249,7 +249,7 @@ void SubstitutionGoal::finished() } /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; + state = &PathSubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } @@ -278,12 +278,12 @@ void SubstitutionGoal::finished() } -void SubstitutionGoal::handleChildOutput(int fd, const string & data) +void PathSubstitutionGoal::handleChildOutput(int fd, const string & data) { } -void SubstitutionGoal::handleEOF(int fd) +void PathSubstitutionGoal::handleEOF(int fd) { if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index dee2cecbf..3b3cb7e32 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,7 +8,7 @@ namespace nix { class Worker; -struct SubstitutionGoal : public Goal +struct PathSubstitutionGoal : public Goal { /* The store path that should be realised through a substitute. */ StorePath storePath; @@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (SubstitutionGoal::*GoalState)(); + typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; /* Content address for recomputing store path */ std::optional ca; - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); +public: + PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b2223c3b6..619b1d69c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -78,12 +78,12 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - std::weak_ptr & goal_weak = substitutionGoals[path]; + std::weak_ptr & goal_weak = substitutionGoals[path]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; wakeUp(goal); } @@ -109,7 +109,7 @@ void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); else assert(false); @@ -217,7 +217,7 @@ void Worker::run(const Goals & _topGoals) topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { topPaths.push_back({goal->drvPath, goal->wantedOutputs}); - } else if (auto goal = dynamic_cast(i.get())) { + } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back({goal->storePath}); } } @@ -471,7 +471,7 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 82e711191..42acf8542 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -12,18 +12,18 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; -struct SubstitutionGoal; +struct PathSubstitutionGoal; /* Workaround for not being able to declare a something like - class SubstitutionGoal : public Goal; + class PathSubstitutionGoal : public Goal; even when Goal is a complete type. This is still a static cast. The purpose of exporting it is to define it in - a place where `SubstitutionGoal` is concrete, and use it in a place where it + a place where `PathSubstitutionGoal` is concrete, and use it in a place where it is opaque. */ -GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -72,7 +72,7 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ std::map> derivationGoals; - std::map> substitutionGoals; + std::map> substitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -146,7 +146,7 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 03bb0218d..fc67f215a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,7 +281,9 @@ private: void createUser(const std::string & userName, uid_t userId) override; friend struct LocalDerivationGoal; + friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; + friend struct DerivationGoal; }; From df9d4f88d5aed0aa4ed67eb012e9f260550b7200 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 15:40:10 +0100 Subject: [PATCH 420/477] Allow substituting drv outputs when building --- src/libstore/build/derivation-goal.cc | 25 +++-- .../build/drv-output-substitution-goal.cc | 95 +++++++++++++++++++ .../build/drv-output-substitution-goal.hh | 50 ++++++++++ src/libstore/build/worker.cc | 22 ++++- src/libstore/build/worker.hh | 5 + 5 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 src/libstore/build/drv-output-substitution-goal.cc create mode 100644 src/libstore/build/drv-output-substitution-goal.hh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7b97e575a..7dcd2a6eb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -246,17 +246,22 @@ void DerivationGoal::haveDerivation() through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { + for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(upcast_goal(worker.makePathSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + if (!status.known) + addWaitee( + upcast_goal( + worker.makeDrvOutputSubstitutionGoal( + DrvOutput{status.outputHash, outputName}, + buildMode == bmRepair ? Repair : NoRepair + ) + ) + ); + else + addWaitee(upcast_goal(worker.makePathSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv)))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc new file mode 100644 index 000000000..a5ac4c49d --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -0,0 +1,95 @@ +#include "drv-output-substitution-goal.hh" +#include "worker.hh" +#include "substitution-goal.hh" + +namespace nix { + +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional ca) + : Goal(worker) + , id(id) +{ + state = &DrvOutputSubstitutionGoal::init; + name = fmt("substitution of '%s'", id.to_string()); + trace("created"); +} + + +void DrvOutputSubstitutionGoal::init() +{ + trace("init"); + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + tryNext(); +} + +void DrvOutputSubstitutionGoal::tryNext() +{ + trace("Trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + return; + } + + auto sub = subs.front(); + subs.pop_front(); + + // FIXME: Make async + outputInfo = sub->queryRealisation(id); + if (!outputInfo) { + tryNext(); + return; + } + + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); + + if (waitees.empty()) outPathValid(); + else state = &DrvOutputSubstitutionGoal::outPathValid; +} + +void DrvOutputSubstitutionGoal::outPathValid() +{ + assert(outputInfo); + trace("Output path substituted"); + + if (nrFailed > 0) { + debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + worker.store.registerDrvOutput(*outputInfo); + finished(); +} + +void DrvOutputSubstitutionGoal::finished() +{ + trace("finished"); + amDone(ecSuccess); +} + +string DrvOutputSubstitutionGoal::key() +{ + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + std::string(id.to_string()); +} + +void DrvOutputSubstitutionGoal::work() +{ + (this->*state)(); +} + +} diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh new file mode 100644 index 000000000..63ab53d89 --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -0,0 +1,50 @@ +#pragma once + +#include "store-api.hh" +#include "goal.hh" +#include "realisation.hh" + +namespace nix { + +class Worker; + +// Substitution of a derivation output. +// This is done in three steps: +// 1. Fetch the output info from a substituter +// 2. Substitute the corresponding output path +// 3. Register the output info +class DrvOutputSubstitutionGoal : public Goal { +private: + // The drv output we're trying to substitue + DrvOutput id; + + // The realisation corresponding to the given output id. + // Will be filled once we can get it. + std::optional outputInfo; + + /* The remaining substituters. */ + std::list> subs; + + /* Whether a substituter failed. */ + bool substituterFailed = false; + +public: + DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + + typedef void (DrvOutputSubstitutionGoal::*GoalState)(); + GoalState state; + + void init(); + void tryNext(); + void outPathValid(); + void finished(); + + void timedOut(Error && ex) override { abort(); }; + + string key() override; + + void work() override; + +}; + +} diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 619b1d69c..616b17e61 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -90,8 +91,20 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } -template -static void removeGoal(std::shared_ptr goal, std::map> & goalMap) +std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +{ + std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; + auto goal = goal_weak.lock(); // FIXME + if (!goal) { + goal = std::make_shared(id, *this, repair, ca); + goal_weak = goal; + wakeUp(goal); + } + return goal; +} + +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ for (auto i = goalMap.begin(); @@ -111,6 +124,8 @@ void Worker::removeGoal(GoalPtr goal) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, drvOutputSubstitutionGoals); else assert(false); if (topGoals.find(goal) != topGoals.end()) { @@ -474,5 +489,8 @@ void Worker::markContentsGood(const StorePath & path) GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) { + return subGoal; +} } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 42acf8542..918de35f6 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "lock.hh" #include "store-api.hh" #include "goal.hh" +#include "realisation.hh" #include #include @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; struct PathSubstitutionGoal; +class DrvOutputSubstitutionGoal; /* Workaround for not being able to declare a something like @@ -24,6 +26,7 @@ struct PathSubstitutionGoal; a place where `PathSubstitutionGoal` is concrete, and use it in a place where it is opaque. */ GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -73,6 +76,7 @@ private: same derivation / path. */ std::map> derivationGoals; std::map> substitutionGoals; + std::map> drvOutputSubstitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -147,6 +151,7 @@ public: /* substitution goal */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); From 93b5a59b674c0a29846828c7d14b434cc954f8ee Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 16:04:43 +0100 Subject: [PATCH 421/477] Add a test for the remote caching of CA derivations --- tests/ca/substitute.sh | 21 +++++++++++++++++++++ tests/local.mk | 3 ++- tests/push-to-store.sh | 6 ++++-- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 tests/ca/substitute.sh diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh new file mode 100644 index 000000000..79a6ef8b1 --- /dev/null +++ b/tests/ca/substitute.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Ensure that binary substitution works properly with ca derivations + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf + +export REMOTE_STORE=file://$TEST_ROOT/binary_cache + +buildDrvs () { + nix build --file ./content-addressed.nix -L --no-link "$@" +} + +# Populate the remote cache +buildDrvs --post-build-hook ../push-to-store.sh + +# Restart the build on an empty store, ensuring that we don't build +clearStore +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 + diff --git a/tests/local.mk b/tests/local.mk index 07cfd7a50..e17555051 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -41,7 +41,8 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/nix-copy.sh + ca/nix-copy.sh \ + ca/substitute.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh index 6aadb916b..25352c751 100755 --- a/tests/push-to-store.sh +++ b/tests/push-to-store.sh @@ -1,4 +1,6 @@ #!/bin/sh -echo Pushing "$@" to "$REMOTE_STORE" -printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs +set -x + +echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" +printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs From 9931f18c2dfff2642dea8e1a153eaaa58d7e3c8a Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sun, 21 Feb 2021 11:08:28 -0500 Subject: [PATCH 422/477] Add support for bare git repositories with git+file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local git repositories are normally used directly instead of cloning. This commit checks if a repo is bare and forces a clone. Co-authored-by: Théophane Hufschmitt --- src/libfetchers/git.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 81c647f89..4f9db1bcd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -153,12 +153,14 @@ struct GitInputScheme : InputScheme std::pair getActualUrl(const Input & input) const { - // Don't clone file:// URIs (but otherwise treat them the - // same as remote URIs, i.e. don't use the working tree or - // HEAD). + // file:// URIs are normally not cloned (but otherwise treated the + // same as remote URIs, i.e. we don't use the working tree or + // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git + // repo, treat as a remote URI to force a clone. static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); - bool isLocal = url.scheme == "file" && !forceHttp; + bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); + bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; return {isLocal, isLocal ? url.path : url.base}; } From 92a234322f5a46b65825c748220cef40209eeacd Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sun, 21 Feb 2021 10:41:46 -0500 Subject: [PATCH 423/477] Add test for git+file with bare repository --- tests/flakes.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 25ba2ac43..9747aba7a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -25,6 +25,7 @@ templatesDir=$TEST_ROOT/templates nonFlakeDir=$TEST_ROOT/nonFlake flakeA=$TEST_ROOT/flakeA flakeB=$TEST_ROOT/flakeB +flakeGitBare=$TEST_ROOT/flakeGitBare for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do rm -rf $repo $repo.tmp @@ -604,6 +605,11 @@ nix flake update $flake3Dir [[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]] [[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]] +# Test git+file with bare repo. +rm -rf $flakeGitBare +git clone --bare $flake1Dir $flakeGitBare +nix build -o $TEST_ROOT/result git+file://$flakeGitBare + # Test Mercurial flakes. rm -rf $flake5Dir hg init $flake5Dir From 7ce10924c74e9e037b05558aeb5f0639df5955f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Mar 2021 15:07:09 +0000 Subject: [PATCH 424/477] Fix bad wanted output error as requested - UsageError -> Error - include drv path too --- src/libstore/build/derivation-goal.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 530f8829a..4c3bccf25 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1274,7 +1274,9 @@ void DerivationGoal::checkPathValidity() // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) - throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft)); + throw Error("derivation '%s' does not have wanted outputs %s", + worker.store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); } From fc6bfb261d50102016ed812ecf9949d41fe539f7 Mon Sep 17 00:00:00 2001 From: dramforever Date: Tue, 2 Mar 2021 21:56:50 +0800 Subject: [PATCH 425/477] libfetchers/tarball: Lock on effectiveUrl Basically, if a tarball URL is used as a flake input, and the URL leads to a redirect, the final redirect destination would be recorded as the locked URL. This allows tarballs under https://nixos.org/channels to be used as flake inputs. If we, as before, lock on to the original URL it would break every time the channel updates. --- src/libfetchers/fetchers.hh | 8 +++++++- src/libfetchers/github.cc | 6 +++--- src/libfetchers/tarball.cc | 19 ++++++++++++++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a72cfafa4..c6b219c02 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -145,7 +145,13 @@ DownloadFileResult downloadFile( bool immutable, const Headers & headers = {}); -std::pair downloadTarball( +struct DownloadTarballMeta +{ + time_t lastModified; + std::string effectiveUrl; +}; + +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8352ef02d..3e5ad75a8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); + input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", uint64_t(lastModified)} + {"lastModified", uint64_t(meta.lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b8d7d2c70..bd05bb2f1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -109,7 +109,7 @@ DownloadFileResult downloadFile( }; } -std::pair downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -127,7 +127,10 @@ std::pair downloadTarball( if (cached && !cached->expired) return { Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), - getIntAttr(cached->infoAttrs, "lastModified") + { + .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")), + .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url), + }, }; auto res = downloadFile(store, url, name, immutable, headers); @@ -152,6 +155,7 @@ std::pair downloadTarball( Attrs infoAttrs({ {"lastModified", uint64_t(lastModified)}, + {"effectiveUrl", res.effectiveUrl}, {"etag", res.etag}, }); @@ -164,7 +168,10 @@ std::pair downloadTarball( return { Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), - lastModified, + { + .lastModified = lastModified, + .effectiveUrl = res.effectiveUrl, + }, }; } @@ -223,9 +230,11 @@ struct TarballInputScheme : InputScheme return true; } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & _input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + Input input(_input); + auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false); + input.attrs.insert_or_assign("url", meta.effectiveUrl); return {std::move(tree), input}; } }; From 7331da99abead2b59efcfdaf729cb1034642b630 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Feb 2021 13:35:31 +0100 Subject: [PATCH 426/477] Make NIX_SHOW_STATS work with new-style commands --- src/libcmd/command.hh | 2 ++ src/libcmd/installables.cc | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index c02193924..e66c697eb 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -48,6 +48,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs ref getEvalState(); std::shared_ptr evalState; + + ~EvalCommand(); }; struct MixFlakeOptions : virtual Args, EvalCommand diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4739dc974..7102f5a1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -280,6 +280,12 @@ ref EvalCommand::getEvalState() return ref(evalState); } +EvalCommand::~EvalCommand() +{ + if (evalState) + evalState->printStats(); +} + void completeFlakeRef(ref store, std::string_view prefix) { if (prefix == "") From 665d4ec2dac6734caff9de5b030be123cb7276ef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Mar 2021 17:52:57 +0100 Subject: [PATCH 427/477] nix repl :doc: Don't return docs for partially applied primops This gives misleading results for Nixpkgs functions like lib.toUpper. Fixes #4596. --- src/libexpr/eval.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e2f2308aa..3afe2e47b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -592,10 +592,8 @@ Value & EvalState::getBuiltin(const string & name) std::optional EvalState::getDoc(Value & v) { - if (v.isPrimOp() || v.isPrimOpApp()) { + if (v.isPrimOp()) { auto v2 = &v; - while (v2->isPrimOpApp()) - v2 = v2->primOpApp.left; if (v2->primOp->doc) return Doc { .pos = noPos, From e16431b4665c0362f66bace7734fed0a6c0692d5 Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 4 Mar 2021 16:14:23 +0700 Subject: [PATCH 428/477] improve man page for nix.conf (builders) --- src/libstore/globals.hh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a51d9c2f1..bf0767dfa 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -206,7 +206,17 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of `nix.machines`."}; + R"( + A semicolon-separated list of build machines, where each machine follows this format: + + {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}] + + Examples: + + ssh://root@builder1.com + + ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm + )"}; Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", From 6212e89bf604d61fc896f21f66908be6fbbfcc5d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 5 Mar 2021 00:49:46 +0000 Subject: [PATCH 429/477] Avoid some StorePath -> Path -> StorePath roundtrips There were done when StorePath was defined in Rust and there were some FFI issues. This is no longer an issue. --- src/libstore/misc.cc | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..f58816ad8 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -22,55 +22,53 @@ void Store::computeFSClosure(const StorePathSet & startPaths, Sync state_(State{0, paths_, 0}); - std::function enqueue; + std::function enqueue; std::condition_variable done; - enqueue = [&](const Path & path) -> void { + enqueue = [&](const StorePath & path) -> void { { auto state(state_.lock()); if (state->exc) return; - if (!state->paths.insert(parseStorePath(path)).second) return; + if (!state->paths.insert(path).second) return; state->pending++; } - queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future> fut) { + queryPathInfo(path, {[&](std::future> fut) { // FIXME: calls to isValidPath() should be async try { auto info = fut.get(); - auto path = parseStorePath(pathS); - if (flipDirection) { StorePathSet referrers; queryReferrers(path, referrers); for (auto & ref : referrers) if (ref != path) - enqueue(printStorePath(ref)); + enqueue(ref); if (includeOutputs) for (auto & i : queryValidDerivers(path)) - enqueue(printStorePath(i)); + enqueue(i); if (includeDerivers && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(printStorePath(i)); + enqueue(i); } else { for (auto & ref : info->references) if (ref != path) - enqueue(printStorePath(ref)); + enqueue(ref); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(printStorePath(i)); + if (isValidPath(i)) enqueue(i); if (includeDerivers && info->deriver && isValidPath(*info->deriver)) - enqueue(printStorePath(*info->deriver)); + enqueue(*info->deriver); } @@ -90,7 +88,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, }; for (auto & startPath : startPaths) - enqueue(printStorePath(startPath)); + enqueue(startPath); { auto state(state_.lock()); @@ -160,13 +158,10 @@ void Store::queryMissing(const std::vector & targets, }; auto checkOutput = [&]( - const Path & drvPathS, ref drv, const Path & outPathS, ref> drvState_) + const StorePath & drvPath, ref drv, const StorePath & outPath, ref> drvState_) { if (drvState_->lock()->done) return; - auto drvPath = parseStorePath(drvPathS); - auto outPath = parseStorePath(outPathS); - SubstitutablePathInfos infos; querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); @@ -203,7 +198,7 @@ void Store::queryMissing(const std::vector & targets, return; } - PathSet invalid; + StorePathSet invalid; /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; @@ -213,7 +208,7 @@ void Store::queryMissing(const std::vector & targets, break; } if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) - invalid.insert(printStorePath(*pathOpt)); + invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; @@ -223,7 +218,7 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); } else mustBuildDrv(path.path, *drv); From 6e849e3b0a6eb46e6dc65cbd091cc829eab09a5f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 3 Mar 2021 14:46:15 -0800 Subject: [PATCH 430/477] nix-build: set execfail When starting a nix-shell with `-i` it was previously possible for it to silently fail in the scenario where the specified interpreter didn't exist. This happened due to the `exec` call masking the issue. With this change we enable `execfail`, which causes the script using `nix-shell` as interpreter to correctly exit with code 127. Fixes: #4598 --- src/nix-build/nix-build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7b4a53919..65b85b304 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -447,6 +447,7 @@ static void main_nix_build(int argc, char * * argv) "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " "unset TZ; %6%" + "shopt -s execfail;" "%7%", shellEscape(tmpDir), (pure ? "" : "p=$PATH; "), From ac8ba2eae4fc649d7a3a19815631b4d76e60d74a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sat, 6 Mar 2021 19:51:29 -0600 Subject: [PATCH 431/477] remove doc for obsolete --no-build-hook flag `--no-build-hook` appears to have been removed in 25f32625e2f2a3a1e1b3a3811da82f21c3a3b880 --- doc/manual/src/command-ref/opt-common.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 9650f53f8..bc8eb6796 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -134,15 +134,6 @@ Most Nix commands accept the following command-line options: failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). - - `--no-build-hook` - Disables the build hook mechanism. This allows to ignore remote - builders if they are setup on the machine. - - It's useful in cases where the bandwidth between the client and the - remote builder is too low. In that case it can take more time to - upload the sources to the remote builder and fetch back the result - than to do the computation locally. - - `--readonly-mode` When this option is used, no attempt is made to open the Nix database. Most Nix operations do need database access, so those From 89013bdd7ed4007871cc421315b51b7cada8edff Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 9 Mar 2021 10:11:25 +0100 Subject: [PATCH 432/477] Add a `nix realisation` command for working on realisations Currently only has `nix realisation info`, more to come probably --- src/nix/realisation.cc | 78 +++++++++++++++++++++++++++++++++++++ src/nix/realisation/info.md | 15 +++++++ 2 files changed, 93 insertions(+) create mode 100644 src/nix/realisation.cc create mode 100644 src/nix/realisation/info.md diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc new file mode 100644 index 000000000..9ee9ccb91 --- /dev/null +++ b/src/nix/realisation.cc @@ -0,0 +1,78 @@ +#include "command.hh" +#include "common-args.hh" + +#include + +using namespace nix; + +struct CmdRealisation : virtual NixMultiCommand +{ + CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + { } + + std::string description() override + { + return "manipulate a Nix realisation"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix realisation' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRealisation = registerCommand("realisation"); + +struct CmdRealisationInfo : RealisedPathsCommand, MixJSON +{ + std::string description() override + { + return "query information about one or several realisations"; + } + + std::string doc() override + { + return + #include "realisation/info.md" + ; + } + + Category category() override { return catSecondary; } + + void run(ref store, std::vector paths) override + { + settings.requireExperimentalFeature("ca-derivations"); + if (json) { + nlohmann::json res = nlohmann::json::array(); + for (auto & path : paths) { + nlohmann::json currentPath; + if (auto realisation = std::get_if(&path.raw)) + currentPath = realisation->toJSON(); + else + currentPath["opaquePath"] = store->printStorePath(path.path()); + + res.push_back(currentPath); + } + std::cout << res.dump(); + } + else { + for (auto & path : paths) { + if (auto realisation = std::get_if(&path.raw)) { + std::cout << + realisation->id.to_string() << " " << + store->printStorePath(realisation->outPath); + } else + std::cout << store->printStorePath(path.path()); + + std::cout << std::endl; + } + } + } +}; + +static auto rCmdRealisationInfo = registerCommand2({"realisation", "info"}); diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md new file mode 100644 index 000000000..852240f44 --- /dev/null +++ b/src/nix/realisation/info.md @@ -0,0 +1,15 @@ +R"MdBoundary( +# Description + +Display some informations about the given realisation + +# Examples + +Show some information about the realisation of the `hello` package: + +```console +$ nix realisation info nixpkgs#hello --json +[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}] +``` + +)MdBoundary" From 0431cf6d0992e7986afbb3d0ffd0a7e1cca8ae8a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sun, 7 Feb 2021 15:34:24 -0600 Subject: [PATCH 433/477] fix nixbld user name/uid for macOS --- scripts/bigsur-nixbld-user-migration.sh | 46 +++++++++++++++++++++++++ scripts/install-darwin-multi-user.sh | 2 ++ scripts/install-multi-user.sh | 6 ++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100755 scripts/bigsur-nixbld-user-migration.sh diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh new file mode 100755 index 000000000..f1619fd56 --- /dev/null +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +((NEW_NIX_FIRST_BUILD_UID=301)) + +id_available(){ + dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +} + +change_nixbld_names_and_ids(){ + local name uid next_id + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + echo "Attempting to migrate nixbld users." + echo "Each user should change from nixbld# to _nixbld#" + echo "and their IDs relocated to $next_id+" + while read -r name uid; do + echo " Checking $name (uid: $uid)" + # iterate for a clean ID + while id_available "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." + exit 1 + fi + done + + if [[ $name == _* ]]; then + echo " It looks like $name has already been renamed--skipping." + else + # first 3 are cleanup, it's OK if they aren't here + sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + # remove existing user from group + sudo dseditgroup -o edit -t user -d $name nixbld || true + sudo dscl . change /Users/$name UniqueID $uid $next_id + sudo dscl . change /Users/$name RecordName $name _$name + # add renamed user to group + sudo dseditgroup -o edit -t user -a _$name nixbld + echo " $name migrated to _$name (uid: $next_id)" + fi + done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) +} + +change_nixbld_names_and_ids diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index a27be2a43..f6575ae2f 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -4,6 +4,8 @@ set -eu set -o pipefail readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +NIX_FIRST_BUILD_UID="301" +NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" dsclattr() { /usr/bin/dscl . -read "$1" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5e8b4ac18..30ccf1764 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,7 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" +# darwin installer needs to override these +NIX_FIRST_BUILD_UID="30001" +NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -104,7 +106,7 @@ EOF } nix_user_for_core() { - printf "nixbld%d" "$1" + printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1" } nix_uid_for_core() { From 826877cabf9374e0acd5408c6975ee332b1cccc8 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 11:56:33 +0100 Subject: [PATCH 434/477] Add some logic for signing realisations Not exposed anywhere, but built realisations are now signed (and this should be forwarded when copy-ing them around) --- src/libstore/build/local-derivation-goal.cc | 12 ++++-- src/libstore/ca-specific-schema.sql | 1 + src/libstore/local-store.cc | 29 ++++++++++--- src/libstore/local-store.hh | 4 +- src/libstore/realisation.cc | 46 ++++++++++++++++++++- src/libstore/realisation.hh | 8 ++++ src/libstore/store-api.hh | 5 +++ 7 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c2f1dda6..048135ccf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2615,10 +2615,14 @@ void LocalDerivationGoal::registerOutputs() but it's fine to do in all cases. */ if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) - worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, - .outPath = newInfo.path}); + for (auto& [outputName, newInfo] : infos) { + auto thisRealisation = Realisation{ + .id = DrvOutput{initialOutputs.at(outputName).outputHash, + outputName}, + .outPath = newInfo.path}; + getLocalStore().signRealisation(thisRealisation); + worker.store.registerDrvOutput(thisRealisation); + } } } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 93c442826..20ee046a1 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -6,6 +6,7 @@ create table if not exists Realisations ( drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, + signatures text, -- space-separated list primary key (drvPath, outputName), foreign key (outputPath) references ValidPaths(id) on delete cascade ); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 90fb4a4bd..6bc963f27 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -310,13 +310,13 @@ LocalStore::LocalStore(const Params & params) if (settings.isExperimentalFeatureEnabled("ca-derivations")) { state->stmts->RegisterRealisedOutput.create(state->db, R"( - insert or replace into Realisations (drvPath, outputName, outputPath) - values (?, ?, (select id from ValidPaths where path = ?)) + insert or replace into Realisations (drvPath, outputName, outputPath, signatures) + values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path from Realisations + select Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -662,6 +662,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) .exec(); }); } @@ -1107,6 +1108,11 @@ bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) return requireSigs && !info.checkSignatures(*this, getPublicKeys()); } +bool LocalStore::realisationIsUntrusted(const Realisation & realisation) +{ + return requireSigs && !realisation.checkSignatures(getPublicKeys()); +} + void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -1612,6 +1618,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si } +void LocalStore::signRealisation(Realisation & realisation) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + realisation.sign(secretKey); + } +} + void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. @@ -1649,8 +1667,9 @@ std::optional LocalStore::queryRealisation( if (!use.next()) return std::nullopt; auto outputPath = parseStorePath(use.getStr(0)); - return Ret{ - Realisation{.id = id, .outPath = outputPath}}; + auto signatures = tokenizeString(use.getStr(1)); + return Ret{Realisation{ + .id = id, .outPath = outputPath, .signatures = signatures}}; }); } } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fc67f215a..d54609f01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -137,6 +137,7 @@ public: SubstitutablePathInfos & infos) override; bool pathInfoIsTrusted(const ValidPathInfo &) override; + bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -272,9 +273,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo using the secret keys + /* Add signatures to a ValidPathInfo or Realisation using the secret keys specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); + void signRealisation(Realisation &); Path getRealStoreDir() override { return realStoreDir; } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index cd74af4ee..638065547 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -25,27 +25,69 @@ nlohmann::json Realisation::toJSON() const { return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, + {"signatures", signatures}, }; } Realisation Realisation::fromJSON( const nlohmann::json& json, const std::string& whence) { - auto getField = [&](std::string fieldName) -> std::string { + auto getOptionalField = [&](std::string fieldName) -> std::optional { auto fieldIterator = json.find(fieldName); if (fieldIterator == json.end()) + return std::nullopt; + return *fieldIterator; + }; + auto getField = [&](std::string fieldName) -> std::string { + if (auto field = getOptionalField(fieldName)) + return *field; + else throw Error( "Drv output info file '%1%' is corrupt, missing field %2%", whence, fieldName); - return *fieldIterator; }; + StringSet signatures; + if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) + signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), + .signatures = signatures, }; } +std::string Realisation::fingerprint() const +{ + auto serialized = toJSON(); + serialized.erase("signatures"); + return serialized.dump(); +} + +void Realisation::sign(const SecretKey & secretKey) +{ + signatures.insert(secretKey.signDetached(fingerprint())); +} + +bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + +size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const +{ + // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to + // an input-addressed one − because in that case the drv is enough to check + // it − but we can't know that here. + + size_t good = 0; + for (auto & sig : signatures) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index fc92d3c17..f5049c9e9 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,7 @@ #include "path.hh" #include #include "comparator.hh" +#include "crypto.hh" namespace nix { @@ -25,9 +26,16 @@ struct Realisation { DrvOutput id; StorePath outPath; + StringSet signatures; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + std::string fingerprint() const; + void sign(const SecretKey &); + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + size_t checkSignatures(const PublicKeys & publicKeys) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 71a28eeb8..0cd56d34e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -389,6 +389,11 @@ public: return true; } + virtual bool realisationIsUntrusted(const Realisation & ) + { + return true; + } + protected: virtual void queryPathInfoUncached(const StorePath & path, From 3e6017f911127555cfbed71fe4a4df8f70d08bbb Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 15:07:33 +0100 Subject: [PATCH 435/477] pathInfoIsTrusted -> pathInfoIsUntrusted I guess the rationale behind the old name wath that `pathInfoIsTrusted(info)` returns `true` iff we would need to `blindly` trust the path (because it has no valid signature and `requireSigs` is set), but I find it to be a really confusing footgun because it's quite natural to give it the opposite meaning. --- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/local-store.hh | 2 +- src/libstore/store-api.hh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 5d88b8758..7b1ac126e 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -142,7 +142,7 @@ void PathSubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("substituter '%s' does not have a valid signature for path '%s'", sub->getUri(), worker.store.printStorePath(storePath)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6bc963f27..950a9f74e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1103,7 +1103,7 @@ const PublicKeys & LocalStore::getPublicKeys() return *state->publicKeys; } -bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { return requireSigs && !info.checkSignatures(*this, getPublicKeys()); } @@ -1116,7 +1116,7 @@ bool LocalStore::realisationIsUntrusted(const Realisation & realisation) void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (checkSigs && pathInfoIsTrusted(info)) + if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d54609f01..c311d295a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -136,7 +136,7 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - bool pathInfoIsTrusted(const ValidPathInfo &) override; + bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0cd56d34e..b90aeaa4c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -384,7 +384,7 @@ public: we don't really want to add the dependencies listed in a nar info we don't trust anyyways. */ - virtual bool pathInfoIsTrusted(const ValidPathInfo &) + virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; } From 54ced9072b94515a756e1e8e76c92a42f0ccf366 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 16:43:11 +0100 Subject: [PATCH 436/477] Check the signatures when copying store paths around Broken atm --- src/libstore/local-store.cc | 8 ++++++++ src/libstore/local-store.hh | 1 + src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 ++ tests/ca/signatures.sh | 39 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 3 ++- 6 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/ca/signatures.sh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 950a9f74e..83daa7506 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -652,6 +652,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } } +void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) +{ + settings.requireExperimentalFeature("ca-derivations"); + if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) + registerDrvOutput(info); + else + throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); +} void LocalStore::registerDrvOutput(const Realisation & info) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c311d295a..26e034a82 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,7 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ void registerDrvOutput(const Realisation & info) override; + void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); std::optional queryRealisation(const DrvOutput&) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77c310988..5e321cedf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -798,7 +798,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation); + dstStore->registerDrvOutput(realisation, checkSigs); } } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b90aeaa4c..5d19e8949 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -485,6 +485,8 @@ public: */ virtual void registerDrvOutput(const Realisation & output) { unsupported("registerDrvOutput"); } + virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) + { return registerDrvOutput(output); } /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; diff --git a/tests/ca/signatures.sh b/tests/ca/signatures.sh new file mode 100644 index 000000000..4b4e468f7 --- /dev/null +++ b/tests/ca/signatures.sh @@ -0,0 +1,39 @@ +source common.sh + +# Globally enable the ca derivations experimental flag +sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" + +clearStore +clearCache + +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +pk1=$(cat $TEST_ROOT/pk1) + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +ensureCorrectlyCopied () { + attrPath="$1" + nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath" +} + +testOneCopy () { + clearStore + rm -rf "$REMOTE_STORE_DIR" + + attrPath="$1" + nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + --secret-key-files "$TEST_ROOT/sk1" + + ensureCorrectlyCopied "$attrPath" + + # Ensure that we can copy back what we put in the store + clearStore + nix copy --from $REMOTE_STORE \ + --file ./content-addressed.nix "$attrPath" \ + --trusted-public-keys $pk1 +} + +for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do + testOneCopy "$attrPath" +done diff --git a/tests/local.mk b/tests/local.mk index e17555051..9a227bec5 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -41,8 +41,9 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/nix-copy.sh \ ca/substitute.sh + ca/signatures.sh \ + ca/nix-copy.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 703c98c6cb922ff9d8cd8cb2c1104e0d3b15b803 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 17:32:20 +0100 Subject: [PATCH 437/477] Properly sign the unresolved drvs Don't let them inherit the signature from the parent one (because it makes no sense to do so), but re-sign them after they have been built --- src/libstore/build/derivation-goal.cc | 2 ++ src/libstore/build/derivation-goal.hh | 3 +++ src/libstore/build/local-derivation-goal.cc | 7 ++++++- src/libstore/build/local-derivation-goal.hh | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7dcd2a6eb..d624e58b9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -925,6 +925,8 @@ void DerivationGoal::resolvedFinished() { if (realisation) { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; + newRealisation.signatures.clear(); + signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index c85bcd84f..704b77caf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -180,6 +180,9 @@ struct DerivationGoal : public Goal /* Open a log file and a pipe to it. */ Path openLogFile(); + /* Sign the newly built realisation if the store allows it */ + virtual void signRealisation(Realisation&) {} + /* Close the log file. */ void closeLogFile(); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 048135ccf..2966bb565 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2620,12 +2620,17 @@ void LocalDerivationGoal::registerOutputs() .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, .outPath = newInfo.path}; - getLocalStore().signRealisation(thisRealisation); + signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } } } +void LocalDerivationGoal::signRealisation(Realisation & realisation) +{ + getLocalStore().signRealisation(realisation); +} + void LocalDerivationGoal::checkOutputs(const std::map & outputs) { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 4bbf27a1b..47b818a8b 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -161,6 +161,8 @@ struct LocalDerivationGoal : public DerivationGoal as valid. */ void registerOutputs() override; + void signRealisation(Realisation &) override; + /* Check that an output meets the requirements specified by the 'outputChecks' attribute (or the legacy '{allowed,disallowed}{References,Requisites}' attributes). */ From 5869b3025d8ed2b99a8dca61f335789ce6dc83e1 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 16 Mar 2021 02:42:14 +0100 Subject: [PATCH 438/477] tests/local.mk: fix missing newline escape Fixes syntax error introduced in 54ced9072b94515a756e1e8e76c92a42f0ccf366. --- tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index 1ca363091..de095c117 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -42,7 +42,7 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/substitute.sh + ca/substitute.sh \ ca/signatures.sh \ ca/nix-copy.sh # parallel.sh From 5716345adf2e794fd62229ea52352e74e92e8e63 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 10 Nov 2020 10:43:33 +0100 Subject: [PATCH 439/477] Add a test ensuring compatibility with an old daemon This requires adding `nix` to its own closure which is a bit unfortunate, but as it is optional (the test will be disabled if `OUTER_NIX` is unset) it shouldn't be too much of an issue. (Ideally this should go in another derivation so that we can build Nix and run the test independently, but as the tests are running in the same derivation as the build it's a bit complicated to do so). --- flake.nix | 9 +++++++++ tests/common.sh.in | 3 +-- tests/local.mk | 2 +- tests/remote-store-old-daemon.sh | 7 +++++++ tests/remote-store.sh | 4 ++-- 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/remote-store-old-daemon.sh diff --git a/flake.nix b/flake.nix index e59ec9a35..3e236aaca 100644 --- a/flake.nix +++ b/flake.nix @@ -150,6 +150,11 @@ # 'nix.perl-bindings' packages. overlay = final: prev: { + # An older version of Nix to test against when using the daemon. + # Currently using `nixUnstable` as the stable one doesn't respect + # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. + mainstream-nix = prev.nixUnstable; + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -158,6 +163,8 @@ VERSION_SUFFIX = versionSuffix; + OUTER_NIX = mainstream-nix; + outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; @@ -486,6 +493,8 @@ stdenv.mkDerivation { name = "nix"; + OUTER_NIX = mainstream-nix; + outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; diff --git a/tests/common.sh.in b/tests/common.sh.in index de44a4da4..277dd6dfa 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -57,7 +57,6 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" - nix-store --init clearProfiles } @@ -73,7 +72,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix daemon & + ${NIX_DAEMON_COMMAND:-nix daemon} & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 diff --git a/tests/local.mk b/tests/local.mk index de095c117..dd9a0ad56 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - local-store.sh remote-store.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh new file mode 100644 index 000000000..ede7ce716 --- /dev/null +++ b/tests/remote-store-old-daemon.sh @@ -0,0 +1,7 @@ +# Test that the new Nix can properly talk to an old daemon. +# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test + +if [[ -n "$OUTER_NIX" ]]; then + export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon + source remote-store.sh +fi diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f7ae1a2ed..31210ab47 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -23,12 +23,12 @@ startDaemon storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh +nix-store --gc --max-freed 1K + nix-store --dump-db > $TEST_ROOT/d1 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 cmp $TEST_ROOT/d1 $TEST_ROOT/d2 -nix-store --gc --max-freed 1K - killDaemon user=$(whoami) From eab9cdbd75e739be33f9433cfba9ab354d084440 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 17 Nov 2020 14:33:09 +0100 Subject: [PATCH 440/477] Add a test for the migration of the db between versions --- tests/db-migration.sh | 25 +++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 26 insertions(+) create mode 100644 tests/db-migration.sh diff --git a/tests/db-migration.sh b/tests/db-migration.sh new file mode 100644 index 000000000..e6a405770 --- /dev/null +++ b/tests/db-migration.sh @@ -0,0 +1,25 @@ +# Test that we can successfully migrate from an older db schema + +# Only run this if we have an older Nix available +if [[ -z "$OUTER_NIX" ]]; then + exit 0 +fi + +source common.sh + +# Fill the db using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$OUTER_NIX/bin:$PATH" +clearStore +nix-build simple.nix --no-out-link +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +dependenciesOutPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1") +fixedOutPath=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build fixed.nix -A good.0 --no-out-link) + +# Migrate to the new schema and ensure that everything's there +export PATH="$PATH_WITH_NEW_NIX" +info=$(nix path-info --json $dependenciesOutPath) +[[ $info =~ '"ultimate":true' ]] +[[ $info =~ 'cache1.example.org' ]] +nix verify -r "$fixedOutPath" +nix verify -r "$dependenciesOutPath" --sigs-needed 1 --trusted-public-keys $(cat $TEST_ROOT/pk1) diff --git a/tests/local.mk b/tests/local.mk index dd9a0ad56..01c35551f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -7,6 +7,7 @@ nix_tests = \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ + db-migration.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ From a0866c8ea4bc66f9aacc7ad19139d57946b3df18 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 13:43:08 +0100 Subject: [PATCH 441/477] Make the tests (optionnally) run in another derivation That way we can run them without rebuilding Nix --- flake.nix | 41 ++++++++++++++++++++++++++++++++++++----- tests/common.sh.in | 6 ++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 3e236aaca..c2e5db53a 100644 --- a/flake.nix +++ b/flake.nix @@ -144,6 +144,32 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; + testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { + NIX_DAEMON_PACKAGE = daemon; + NIX_CLIENT_PACKAGE = client; + name = "nix-tests-${client.version}-against-${daemon.version}"; + inherit version; + + src = self; + + VERSION_SUFFIX = versionSuffix; + + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ awsDeps; + propagatedBuildInputs = propagatedDeps; + + enableParallelBuilding = true; + + dontBuild = true; + doInstallCheck = true; + + installPhase = '' + mkdir -p $out + ''; + installCheckPhase = "make installcheck"; + + }; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -153,7 +179,7 @@ # An older version of Nix to test against when using the daemon. # Currently using `nixUnstable` as the stable one doesn't respect # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. - mainstream-nix = prev.nixUnstable; + nixStable = prev.nix; nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; @@ -163,8 +189,6 @@ VERSION_SUFFIX = versionSuffix; - OUTER_NIX = mainstream-nix; - outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; @@ -441,6 +465,15 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + installTests = + let pkgs = nixpkgsFor.${system}; in + pkgs.runCommand "install-tests" { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"; }); packages = forAllSystems (system: { @@ -493,8 +526,6 @@ stdenv.mkDerivation { name = "nix"; - OUTER_NIX = mainstream-nix; - outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; diff --git a/tests/common.sh.in b/tests/common.sh.in index 277dd6dfa..d31d3fbb8 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -29,6 +29,12 @@ unset XDG_CACHE_HOME mkdir -p $TEST_HOME export PATH=@bindir@:$PATH +if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then + export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH +fi +if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then + export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon" +fi coreutils=@coreutils@ export dot=@dot@ From 81df1b5c687b7606f0159485c33bf5f7e2614eba Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 14:15:57 +0100 Subject: [PATCH 442/477] Remove the `remote-store-old-daemon` test Doesn't make sense anymore with the new setup --- tests/local.mk | 2 +- tests/remote-store-old-daemon.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 tests/remote-store-old-daemon.sh diff --git a/tests/local.mk b/tests/local.mk index 01c35551f..e7e85f97e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh export.sh export-graph.sh \ db-migration.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh deleted file mode 100644 index ede7ce716..000000000 --- a/tests/remote-store-old-daemon.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Test that the new Nix can properly talk to an old daemon. -# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test - -if [[ -n "$OUTER_NIX" ]]; then - export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon - source remote-store.sh -fi From be60c9ef50bf5fa653138802f63727fa0aadf50a Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 14:20:10 +0100 Subject: [PATCH 443/477] Fix the db-migration test --- tests/db-migration.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/db-migration.sh b/tests/db-migration.sh index e6a405770..e0ff7d311 100644 --- a/tests/db-migration.sh +++ b/tests/db-migration.sh @@ -1,7 +1,8 @@ # Test that we can successfully migrate from an older db schema # Only run this if we have an older Nix available -if [[ -z "$OUTER_NIX" ]]; then +# XXX: This assumes that the `daemon` package is older than the `client` one +if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then exit 0 fi @@ -9,7 +10,7 @@ source common.sh # Fill the db using the older Nix PATH_WITH_NEW_NIX="$PATH" -export PATH="$OUTER_NIX/bin:$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" clearStore nix-build simple.nix --no-out-link nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 From 5ec873b127139ca90cc31967c25c9a34fb4cc3e4 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 16:44:42 +0100 Subject: [PATCH 444/477] Shorten the test drv name To prevent the OSX build to fail because of a too long socket path --- flake.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index c2e5db53a..1cd54e702 100644 --- a/flake.nix +++ b/flake.nix @@ -147,7 +147,10 @@ testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - name = "nix-tests-${client.version}-against-${daemon.version}"; + # Must keep this name short as OSX has a rather strict limit on the + # socket path length, and this name appears in the path of the + # nix-daemon socket used in the tests + name = "nix-tests"; inherit version; src = self; From 77f5d171e17294ebb017a386d4408bf4613dfed7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Mar 2021 16:53:39 +0100 Subject: [PATCH 445/477] --override-input: Imply --no-write-lock-file Fixes #3779. --- src/libcmd/installables.cc | 3 ++- src/nix/flake.cc | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 7102f5a1a..898e642a5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -111,10 +111,11 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { + lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."))); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2f0c468a8..d37791aba 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -110,6 +110,7 @@ struct CmdFlakeUpdate : FlakeCommand removeFlag("recreate-lock-file"); removeFlag("update-input"); removeFlag("no-update-lock-file"); + removeFlag("no-write-lock-file"); } std::string doc() override @@ -124,6 +125,7 @@ struct CmdFlakeUpdate : FlakeCommand settings.tarballTtl = 0; lockFlags.recreateLockFile = true; + lockFlags.writeLockFile = true; lockFlake(); } @@ -136,6 +138,12 @@ struct CmdFlakeLock : FlakeCommand return "create missing lock file entries"; } + CmdFlakeLock() + { + /* Remove flags that don't make sense. */ + removeFlag("no-write-lock-file"); + } + std::string doc() override { return @@ -147,6 +155,8 @@ struct CmdFlakeLock : FlakeCommand { settings.tarballTtl = 0; + lockFlags.writeLockFile = true; + lockFlake(); } }; From 66fa1c7375e4b3073a16df4678cf1d37446ed20b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Mar 2021 17:19:04 +0100 Subject: [PATCH 446/477] Merge 'nix flake {info,list-inputs}' into 'nix flake metadata' Fixes #4613. --- src/nix/flake-list-inputs.md | 23 ---- src/nix/{flake-info.md => flake-metadata.md} | 27 ++-- src/nix/flake.cc | 134 ++++++++----------- tests/flakes.sh | 38 +++--- 4 files changed, 92 insertions(+), 130 deletions(-) delete mode 100644 src/nix/flake-list-inputs.md rename src/nix/{flake-info.md => flake-metadata.md} (75%) diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md deleted file mode 100644 index 250e13be0..000000000 --- a/src/nix/flake-list-inputs.md +++ /dev/null @@ -1,23 +0,0 @@ -R""( - -# Examples - -* Show the inputs of the `hydra` flake: - - ```console - # nix flake list-inputs github:NixOS/hydra - github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 - ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 - │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f - │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 - └───nixpkgs follows input 'nix/nixpkgs' - ``` - -# Description - -This command shows the inputs of the flake specified by the flake -referenced *flake-url*. Since it prints the locked inputs that result -from generating or updating the lock file, this command essentially -displays the contents of the flake's lock file in human-readable form. - -)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-metadata.md similarity index 75% rename from src/nix/flake-info.md rename to src/nix/flake-metadata.md index fda3171db..5a009409b 100644 --- a/src/nix/flake-info.md +++ b/src/nix/flake-metadata.md @@ -5,19 +5,24 @@ R""( * Show what `nixpkgs` resolves to: ```console - # nix flake info nixpkgs - Resolved URL: github:NixOS/nixpkgs - Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Description: A collection of packages for the Nix package manager - Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source - Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Last modified: 2020-12-23 12:36:12 + # nix flake metadata nixpkgs + Resolved URL: github:edolstra/dwarffs + Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 + Description: A filesystem that fetches DWARF debug info from the Internet on demand + Path: /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source + Revision: f691e2c991e75edb22836f1dbe632c40324215c5 + Last modified: 2021-01-21 15:41:26 + Inputs: + ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' ``` * Show information about `dwarffs` in JSON format: ```console - # nix flake info dwarffs --json | jq . + # nix flake metadata dwarffs --json | jq . { "description": "A filesystem that fetches DWARF debug info from the Internet on demand", "lastModified": 1597153508, @@ -29,6 +34,7 @@ R""( "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", "type": "github" }, + "locks": { ... }, "original": { "id": "dwarffs", "type": "indirect" @@ -75,6 +81,9 @@ data. This includes: time of the commit of the locked flake; for tarball flakes, it's the most recent timestamp of any file inside the tarball. +* `Inputs`: The flake inputs with their corresponding lock file + entries. + With `--json`, the output is a JSON object with the following fields: * `original` and `originalUrl`: The flake reference specified by the @@ -96,4 +105,6 @@ With `--json`, the output is a JSON object with the following fields: * `lastModified`: See `Last modified` above. +* `locks`: The contents of `flake.lock`. + )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d37791aba..5ce2e082c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -43,12 +43,6 @@ public: return parseFlakeRef(flakeUrl, absPath(".")); //FIXME } - Flake getFlake() - { - auto evalState = getEvalState(); - return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); - } - LockedFlake lockFlake() { return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); @@ -60,43 +54,6 @@ public: } }; -static void printFlakeInfo(const Store & store, const Flake & flake) -{ - logger->cout("Resolved URL: %s", flake.resolvedRef.to_string()); - logger->cout("Locked URL: %s", flake.lockedRef.to_string()); - if (flake.description) - logger->cout("Description: %s", *flake.description); - logger->cout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input.getRev()) - logger->cout("Revision: %s", rev->to_string(Base16, false)); - if (auto revCount = flake.lockedRef.input.getRevCount()) - logger->cout("Revisions: %s", *revCount); - if (auto lastModified = flake.lockedRef.input.getLastModified()) - logger->cout("Last modified: %s", - std::put_time(std::localtime(&*lastModified), "%F %T")); -} - -static nlohmann::json flakeToJSON(const Store & store, const Flake & flake) -{ - nlohmann::json j; - if (flake.description) - j["description"] = *flake.description; - j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); - j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); - j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); - if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); - if (auto revCount = flake.lockedRef.input.getRevCount()) - j["revCount"] = *revCount; - if (auto lastModified = flake.lockedRef.input.getLastModified()) - j["lastModified"] = *lastModified; - j["path"] = store.printStorePath(flake.sourceInfo->storePath); - return j; -} - struct CmdFlakeUpdate : FlakeCommand { std::string description() override @@ -175,54 +132,72 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, callback(attr.name, *attr.value, *attr.pos); } -struct CmdFlakeInfo : FlakeCommand, MixJSON +struct CmdFlakeMetadata : FlakeCommand, MixJSON { std::string description() override { - return "list info about a given flake"; + return "show flake metadata"; } std::string doc() override { return - #include "flake-info.md" + #include "flake-metadata.md" ; } void run(nix::ref store) override { - auto flake = getFlake(); + auto lockedFlake = lockFlake(); + auto & flake = lockedFlake.flake; if (json) { - auto json = flakeToJSON(*store, flake); - logger->cout("%s", json.dump()); - } else - printFlakeInfo(*store, flake); - } -}; + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["locks"] = lockedFlake.lockFile.toJSON(); + logger->cout("%s", j.dump()); + } else { + logger->cout( + ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", + flake.resolvedRef.to_string()); + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); + if (flake.description) + logger->cout( + ANSI_BOLD "Description:" ANSI_NORMAL " %s", + *flake.description); + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->cout( + ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", + *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->cout( + ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); -struct CmdFlakeListInputs : FlakeCommand, MixJSON -{ - std::string description() override - { - return "list flake inputs"; - } - - std::string doc() override - { - return - #include "flake-list-inputs.md" - ; - } - - void run(nix::ref store) override - { - auto flake = lockFlake(); - - if (json) - logger->cout("%s", flake.lockFile.toJSON()); - else { - logger->cout("%s", flake.flake.lockedRef); + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; @@ -236,7 +211,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); + *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); bool firstVisit = visited.insert(*lockedNode).second; @@ -249,8 +224,8 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON } }; - visited.insert(flake.lockFile.root); - recurse(*flake.lockFile.root, ""); + visited.insert(lockedFlake.lockFile.root); + recurse(*lockedFlake.lockFile.root, ""); } } }; @@ -1048,8 +1023,7 @@ struct CmdFlake : NixMultiCommand : MultiCommand({ {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, - {"list-inputs", []() { return make_ref(); }}, + {"metadata", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, {"init", []() { return make_ref(); }}, {"new", []() { return make_ref(); }}, diff --git a/tests/flakes.sh b/tests/flakes.sh index 9747aba7a..e78e4a39d 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -164,16 +164,17 @@ EOF # Test 'nix flake list'. [[ $(nix registry list | wc -l) == 7 ]] -# Test 'nix flake info'. -nix flake info flake1 | grep -q 'URL: .*flake1.*' +# Test 'nix flake metadata'. +nix flake metadata flake1 +nix flake metadata flake1 | grep -q 'Locked URL:.*flake1.*' -# Test 'nix flake info' on a local flake. -(cd $flake1Dir && nix flake info) | grep -q 'URL: .*flake1.*' -(cd $flake1Dir && nix flake info .) | grep -q 'URL: .*flake1.*' -nix flake info $flake1Dir | grep -q 'URL: .*flake1.*' +# Test 'nix flake metadata' on a local flake. +(cd $flake1Dir && nix flake metadata) | grep -q 'URL:.*flake1.*' +(cd $flake1Dir && nix flake metadata .) | grep -q 'URL:.*flake1.*' +nix flake metadata $flake1Dir | grep -q 'URL:.*flake1.*' -# Test 'nix flake info --json'. -json=$(nix flake info flake1 --json | jq .) +# Test 'nix flake metadata --json'. +json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] @@ -181,7 +182,7 @@ hash1=$(echo "$json" | jq -r .revision) echo -n '# foo' >> $flake1Dir/flake.nix git -C $flake1Dir commit -a -m 'Foo' -hash2=$(nix flake info flake1 --json --refresh | jq -r .revision) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. @@ -630,7 +631,7 @@ hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit' nix build -o $TEST_ROOT/result hg+file://$flake5Dir [[ -e $TEST_ROOT/result/hello ]] -(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision) nix eval hg+file://$flake5Dir#expr @@ -638,13 +639,13 @@ nix eval hg+file://$flake5Dir#expr (! nix eval hg+file://$flake5Dir#expr --no-allow-dirty) -(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision) hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file' -nix flake info --json hg+file://$flake5Dir --refresh | jq -e -r .revision -nix flake info --json hg+file://$flake5Dir -[[ $(nix flake info --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]] +nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision +nix flake metadata --json hg+file://$flake5Dir +[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]] nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty @@ -654,7 +655,7 @@ tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5 nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz # Building with a tarball URL containing a SRI hash should also work. -url=$(nix flake info --json file://$TEST_ROOT/flake.tar.gz | jq -r .url) +url=$(nix flake metadata --json file://$TEST_ROOT/flake.tar.gz | jq -r .url) [[ $url =~ sha256- ]] nix build -o $TEST_ROOT/result $url @@ -680,9 +681,8 @@ nix flake lock $flake3Dir nix flake lock $flake3Dir --update-input flake2/flake1 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] -# Test 'nix flake list-inputs'. -[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]] -nix flake list-inputs $flake3Dir --json | jq . +# Test 'nix flake metadata --json'. +nix flake metadata $flake3Dir --json | jq . # Test circular flake dependencies. cat > $flakeA/flake.nix < Date: Tue, 16 Mar 2021 18:51:17 +0100 Subject: [PATCH 447/477] Fix Nix to properly work with stores using a scoped IPv6 address According to RFC4007[1], IPv6 addresses can have a so-called zone_id separated from the actual address with `%` as delimiter. In contrast to Nix 2.3, the version on `master` doesn't recognize it as such: $ nix ping-store --store ssh://root@fe80::1%18 --experimental-features nix-command warning: 'ping-store' is a deprecated alias for 'store ping' error: --- Error ----------------------------------------------------------------- nix don't know how to open Nix store 'ssh://root@fe80::1%18' I modified the IPv6 match-regex accordingly to optionally detect this part of the address. As we don't seem to do anything special with it, I decided to leave it as part of the URL for now. Fixes #4490 [1] https://tools.ietf.org/html/rfc4007 --- src/libutil/tests/url.cc | 18 ++++++++++++++++++ src/libutil/url-parts.hh | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index 80646ad3e..aff58e9ee 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -117,6 +117,24 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parseScopedRFC4007IPv6Address) { + auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .scheme = "http", + .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + TEST(parseURL, parseIPv6Address) { auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; auto parsed = parseURL(s); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 862d9fa6e..da10a6bbc 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; -const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+"; +const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; From a61112aadf58c1578cbdcba32b1582d25ca7ed9b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Mar 2021 11:27:11 +0100 Subject: [PATCH 448/477] Remove unimplemented hashAlgoOpt It was in the header but never implemented. --- src/libstore/derivations.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 061d70f69..2df440536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -52,7 +52,7 @@ struct DerivationOutput DerivationOutputCAFloating, DerivationOutputDeferred > output; - std::optional hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer interface provided by BasicDerivation::outputsAndOptPaths */ From ef83ced4e170130cb6f9acd1d253351b02490658 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Mar 2021 17:21:37 +0100 Subject: [PATCH 449/477] Restore 'nix flake info' as a deprecated alias --- src/nix/flake.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5ce2e082c..a2b6c0303 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -230,6 +230,15 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON } }; +struct CmdFlakeInfo : CmdFlakeMetadata +{ + void run(nix::ref store) override + { + warn("'nix flake info' is a deprecated alias for 'nix flake metadata'"); + CmdFlakeMetadata::run(store); + } +}; + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -1024,6 +1033,7 @@ struct CmdFlake : NixMultiCommand {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, {"init", []() { return make_ref(); }}, {"new", []() { return make_ref(); }}, From 3e0e443181997c52b0db19ae781948c573a634dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Mar 2021 17:56:39 +0100 Subject: [PATCH 450/477] ProgressBar: Respect verbosity level This makes its behaviour consistent with SimpleLogger. --- src/libmain/progress-bar.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 0e5432fca..15354549a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -122,6 +122,7 @@ public: void log(Verbosity lvl, const FormatOrString & fs) override { + if (lvl > verbosity) return; auto state(state_.lock()); log(*state, lvl, fs.s); } From 1765711b68c8647b502c2c009dace9632e9300d7 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 18:43:37 -0400 Subject: [PATCH 451/477] tests/config: Fix config test configuration First, "XDG_CONFIG_HOME" shouldn't be named "home", as it may be confusing compared with `$HOME`, which an upcoming test will be using. Then, using a fixed location for the test is problematic. Use `$TEST_ROOT` instead. --- tests/config.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/config.sh b/tests/config.sh index eaa46c395..1edc09c1a 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -1,15 +1,15 @@ source common.sh # Test that files are loaded from XDG by default -export XDG_CONFIG_HOME=/tmp/home -export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +export XDG_CONFIG_HOME=$TEST_ROOT/confighome +export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) -[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] +[[ $files == "$TEST_ROOT/confighome/nix/nix.conf:$TEST_ROOT/dir1/nix/nix.conf:$TEST_ROOT/dir2/nix/nix.conf" ]] # Test that setting NIX_USER_CONF_FILES overrides all the default user config files -export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +export NIX_USER_CONF_FILES=$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) -[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] +[[ $files == "$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf" ]] # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") @@ -24,4 +24,4 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "nix-command flakes" ]] \ No newline at end of file +[[ $exp_features == "nix-command flakes" ]] From bf07581497d55ade85d80e5d9ad9bf5d962e3403 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 19:02:11 -0400 Subject: [PATCH 452/477] tests: Test `.config` stays clean with XDG_CONFIG_HOME set --- tests/config.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/config.sh b/tests/config.sh index 1edc09c1a..01c78f2c3 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -1,5 +1,31 @@ source common.sh +# Isolate the home for this test. +# Other tests (e.g. flake registry tests) could be writing to $HOME in parallel. +export HOME=$TEST_ROOT/userhome + +# Test that using XDG_CONFIG_HOME works +# Assert the config folder didn't exist initially. +[ ! -e "$HOME/.config" ] +# Without XDG_CONFIG_HOME, creates $HOME/.config +unset XDG_CONFIG_HOME +# Run against the nix registry to create the config dir +# (Tip: this relies on removing non-existent entries being a no-op!) +nix registry remove userhome-without-xdg +# Verifies it created it +[ -e "$HOME/.config" ] +# Remove the directory it created +rm -rf "$HOME/.config" +# Run the same test, but with XDG_CONFIG_HOME +export XDG_CONFIG_HOME=$TEST_ROOT/confighome +# Assert the XDG_CONFIG_HOME/nix path does not exist yet. +[ ! -e "$TEST_ROOT/confighome/nix" ] +nix registry remove userhome-with-xdg +# Verifies the confighome path has been created +[ -e "$TEST_ROOT/confighome/nix" ] +# Assert the .config folder hasn't been created. +[ ! -e "$HOME/.config" ] + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 From 66b857244ff062f6bb97c23e2423338ad242f7a1 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 17:56:57 -0400 Subject: [PATCH 453/477] Use the appropriate config dir for the registry --- src/libfetchers/registry.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 81b2227de..74376adc0 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -114,7 +114,7 @@ static std::shared_ptr getSystemRegistry() Path getUserRegistryPath() { - return getHome() + "/.config/nix/registry.json"; + return getConfigDir() + "/nix/registry.json"; } std::shared_ptr getUserRegistry() From 9d309de0de9a09d36717abd02a66b51815397d66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Feb 2021 18:42:46 +0000 Subject: [PATCH 454/477] Clean up serialization for `BuildResult` A few versioning mistakes were corrected: - In 27b5747ca7b5599768083dde5fa4d36bfbb0f66f, Daemon protocol had some version `>= 0xc` that should have been `>= 0x1c`, or `28` since the other conditions used decimal. - In a2b69660a9b326b95d48bd222993c5225bbd5b5f, legacy SSH gated new CAS info on version 6, but version 5 in the server. It is now 6 everywhere. Additionally, legacy ssh was sending over more metadata than the daemon one was. The daemon now sends that data too. CC @regnat Co-authored-by: Cole Helbling --- src/libstore/daemon.cc | 5 ++++- src/libstore/remote-store.cc | 20 ++++++++++++++------ src/libstore/serve-protocol.hh | 2 +- src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 2 +- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ba7959263..dc9cd2cbd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -575,7 +575,10 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; + } + if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { worker_proto::write(*store, to, res.builtOutputs); } break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0d884389a..b01cb5a62 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -62,9 +62,15 @@ void write(const Store & store, Sink & out, const Realisation & realisation) { out << realisation.toJSON().dump(); } DrvOutput read(const Store & store, Source & from, Phantom _) -{ return DrvOutput::parse(readString(from)); } +{ + return DrvOutput::parse(readString(from)); +} + void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ out << drvOutput.to_string(); } +{ + out << drvOutput.to_string(); +} + std::optional read(const Store & store, Source & from, Phantom> _) { @@ -677,10 +683,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->to << buildMode; conn.processStderr(); BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) { + res.status = (BuildResult::Status) readInt(conn->from); + conn->from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); res.builtOutputs = builtOutputs; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 0a17387cb..02d0810cc 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x206 +#define SERVE_PROTOCOL_VERSION (2 << 8 | 6) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 95f08bc9a..be071dd78 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11c +#define PROTOCOL_VERSION (1 << 8 | 29) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94d4881dd..b684feccb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -905,7 +905,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 5)) { + if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { worker_proto::write(*store, out, status.builtOutputs); } From f44206e71953501af502354ab1c747aa2412d676 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Mar 2021 15:18:48 +0000 Subject: [PATCH 455/477] Fix typos in the last PR #4656 --- src/libstore/daemon.cc | 2 +- src/libstore/remote-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index dc9cd2cbd..f28ab6438 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -576,7 +576,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->stopWork(); to << res.status << res.errorMsg; if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; + to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { worker_proto::write(*store, to, res.builtOutputs); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b01cb5a62..ccf095dc2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,7 +686,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); From 0f40561c78bb5566b42d054620b0576e14fe4627 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 23 Mar 2021 10:19:00 +0700 Subject: [PATCH 456/477] nix.conf builders: refer to manual page --- src/libstore/globals.hh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index bf0767dfa..3e4ead76c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -207,15 +207,8 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", R"( - A semicolon-separated list of build machines, where each machine follows this format: - - {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}] - - Examples: - - ssh://root@builder1.com - - ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm + A semicolon-separated list of build machines. + For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; Setting buildersUseSubstitutes{ From 71f92741ec979c1059938a638b7fc8da6d7b0936 Mon Sep 17 00:00:00 2001 From: Nicolas Stig124 FORMICHELLA Date: Tue, 23 Mar 2021 16:23:24 +0100 Subject: [PATCH 457/477] Added Debian-based OS's profiles --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5e8b4ac18..4cc11d210 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -31,7 +31,7 @@ readonly NIX_FIRST_BUILD_UID="30001" readonly NIX_ROOT="/nix" readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" From 4638bcfb2cfb74cb5029c0da0af38bb7ca4b4a6f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Mar 2021 16:14:38 +0100 Subject: [PATCH 458/477] Fix some typos Fixes #4671. --- doc/manual/src/command-ref/nix-store.md | 2 +- src/libexpr/flake/flake.hh | 2 +- src/libstore/globals.cc | 2 +- src/libutil/config.cc | 28 +++++++++++----------- src/libutil/config.hh | 22 ++++++++--------- src/libutil/tests/config.cc | 32 ++++++++++++------------- src/libutil/util.cc | 2 +- src/nix/build.md | 2 +- src/nix/flake-init.md | 2 +- src/nix/flake.md | 4 ++-- src/nix/main.cc | 8 +++---- src/nix/store-prefetch-file.md | 2 +- 12 files changed, 54 insertions(+), 54 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 361c20cc9..49d06f31e 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -79,7 +79,7 @@ paths. Realisation is a somewhat overloaded term: system). If the path is already valid, we are done immediately. Otherwise, the path and any missing paths in its closure may be produced through substitutes. If there are no (successful) - subsitutes, realisation fails. + substitutes, realisation fails. The output path of each derivation is printed on standard output. (For non-derivations argument, the argument itself is printed.) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 65ed1ad0a..d17d5e183 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -113,7 +113,7 @@ struct LockFlags /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; - /* Flake inputs to be overriden. */ + /* Flake inputs to be overridden. */ std::map inputOverrides; /* Flake inputs to be updated. This means that any existing lock diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8d44003f4..d3b27d7be 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -81,7 +81,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverriden(); + globalConfig.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7467e5ac0..bda07cd55 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -20,7 +20,7 @@ bool Config::set(const std::string & name, const std::string & value) return false; } i->second.setting->set(value, append); - i->second.setting->overriden = true; + i->second.setting->overridden = true; return true; } @@ -35,7 +35,7 @@ void Config::addSetting(AbstractSetting * setting) auto i = unknownSettings.find(setting->name); if (i != unknownSettings.end()) { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -48,7 +48,7 @@ void Config::addSetting(AbstractSetting * setting) alias, setting->name); else { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -69,10 +69,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -void Config::getSettings(std::map & res, bool overridenOnly) +void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -136,10 +136,10 @@ void AbstractConfig::applyConfigFile(const Path & path) } catch (SysError &) { } } -void Config::resetOverriden() +void Config::resetOverridden() { for (auto & s : _settings) - s.second.setting->overriden = false; + s.second.setting->overridden = false; } nlohmann::json Config::toJSON() @@ -169,7 +169,7 @@ AbstractSetting::AbstractSetting( void AbstractSetting::setDefault(const std::string & str) { - if (!overriden) set(str); + if (!overridden) set(str); } nlohmann::json AbstractSetting::toJSON() @@ -203,7 +203,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s); }}, + .handler = {[=](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -212,7 +212,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s, true); }}, + .handler = {[=](std::string s) { overridden = true; set(s, true); }}, }); } @@ -365,16 +365,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value) return false; } -void GlobalConfig::getSettings(std::map & res, bool overridenOnly) +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) { for (auto & config : *configRegistrations) - config->getSettings(res, overridenOnly); + config->getSettings(res, overriddenOnly); } -void GlobalConfig::resetOverriden() +void GlobalConfig::resetOverridden() { for (auto & config : *configRegistrations) - config->resetOverriden(); + config->resetOverridden(); } nlohmann::json GlobalConfig::toJSON() diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 71e31656d..bf81b4892 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -71,9 +71,9 @@ public: /** * Adds the currently known settings to the given result map `res`. * - res: map to store settings in - * - overridenOnly: when set to true only overridden settings will be added to `res` + * - overriddenOnly: when set to true only overridden settings will be added to `res` */ - virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + virtual void getSettings(std::map & res, bool overriddenOnly = false) = 0; /** * Parses the configuration in `contents` and applies it @@ -91,7 +91,7 @@ public: /** * Resets the `overridden` flag of all Settings */ - virtual void resetOverriden() = 0; + virtual void resetOverridden() = 0; /** * Outputs all settings to JSON @@ -127,7 +127,7 @@ public: MyClass() : Config(readConfigFile("/etc/my-app.conf")) { - std::cout << foo << "\n"; // will print 123 unless overriden + std::cout << foo << "\n"; // will print 123 unless overridden } }; */ @@ -163,9 +163,9 @@ public: void addSetting(AbstractSetting * setting); - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; @@ -184,7 +184,7 @@ public: int created = 123; - bool overriden = false; + bool overridden = false; void setDefault(const std::string & str); @@ -215,7 +215,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverriden() const { return overriden; } + bool isOverridden() const { return overridden; } }; /* A setting of type T. */ @@ -252,7 +252,7 @@ public: virtual void override(const T & v) { - overriden = true; + overridden = true; value = v; } @@ -324,9 +324,9 @@ struct GlobalConfig : public AbstractConfig bool set(const std::string & name, const std::string & value) override; - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c305af9f5..0ebdaf3db 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -29,20 +29,20 @@ namespace nix { std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, ""); ASSERT_EQ(iter->second.description, "description\n"); } - TEST(Config, getDefinedOverridenSettingNotSet) { + TEST(Config, getDefinedOverriddenSettingNotSet) { Config config; std::string value; std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); const auto e = settings.find("name-of-the-setting"); ASSERT_EQ(e, settings.end()); } @@ -55,7 +55,7 @@ namespace nix { setting.assign("value"); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, "value"); @@ -69,7 +69,7 @@ namespace nix { ASSERT_TRUE(config.set("name-of-the-setting", "value")); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto e = settings.find("name-of-the-setting"); ASSERT_NE(e, settings.end()); ASSERT_EQ(e->second.value, "value"); @@ -100,7 +100,7 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings.find("key"), settings.end()); } @@ -108,17 +108,17 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings["key"].value, "value"); } } - TEST(Config, resetOverriden) { + TEST(Config, resetOverridden) { Config config; - config.resetOverriden(); + config.resetOverridden(); } - TEST(Config, resetOverridenWithSetting) { + TEST(Config, resetOverriddenWithSetting) { Config config; Setting setting{&config, "", "name-of-the-setting", "description"}; @@ -127,7 +127,7 @@ namespace nix { setting.set("foo"); ASSERT_EQ(setting.get(), "foo"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } @@ -135,18 +135,18 @@ namespace nix { std::map settings; setting.override("bar"); - ASSERT_TRUE(setting.overriden); + ASSERT_TRUE(setting.overridden); ASSERT_EQ(setting.get(), "bar"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_FALSE(settings.empty()); } { std::map settings; - config.resetOverriden(); - ASSERT_FALSE(setting.overriden); - config.getSettings(settings, /* overridenOnly = */ true); + config.resetOverridden(); + ASSERT_FALSE(setting.overridden); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ef37275ac..dea9c74b7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1590,7 +1590,7 @@ void startSignalHandlerThread() updateWindowSize(); if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("quering signal mask"); + throw SysError("querying signal mask"); sigset_t set; sigemptyset(&set); diff --git a/src/nix/build.md b/src/nix/build.md index c2f3e387a..20138b7e0 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -81,7 +81,7 @@ path installables are substituted. Unless `--no-link` is specified, after a successful build, it creates symlinks to the store paths of the installables. These symlinks have -the prefix `./result` by default; this can be overriden using the +the prefix `./result` by default; this can be overridden using the `--out-link` option. Each symlink has a suffix `--`, where *N* is the index of the installable (with the left-most installable having index 0), and *outname* is the symbolic derivation output name diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index c66154ad5..890038016 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -24,7 +24,7 @@ R""( This command creates a flake in the current directory by copying the files of a template. It will not overwrite existing files. The default -template is `templates#defaultTemplate`, but this can be overriden +template is `templates#defaultTemplate`, but this can be overridden using `-t`. # Template definitions diff --git a/src/nix/flake.md b/src/nix/flake.md index 440c45dd1..0035195e5 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -70,7 +70,7 @@ Here are some examples of flake references in their URL-like representation: * `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` - entry in the flake registry, with its Git revision overriden to a + entry in the flake registry, with its Git revision overridden to a specific value. * `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` repository on GitHub. @@ -377,7 +377,7 @@ outputs = { self, nixpkgs, grcov }: { }; ``` -Transitive inputs can be overriden from a `flake.nix` file. For +Transitive inputs can be overridden from a `flake.nix` file. For example, the following overrides the `nixpkgs` input of the `nixops` input: diff --git a/src/nix/main.cc b/src/nix/main.cc index 06e221682..f8701ee56 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -309,13 +309,13 @@ void mainWrapped(int argc, char * * argv) if (!args.useNet) { // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overriden) + if (!settings.useSubstitutes.overridden) settings.useSubstitutes = false; - if (!settings.tarballTtl.overriden) + if (!settings.tarballTtl.overridden) settings.tarballTtl = std::numeric_limits::max(); - if (!fileTransferSettings.tries.overriden) + if (!fileTransferSettings.tries.overridden) fileTransferSettings.tries = 0; - if (!fileTransferSettings.connectTimeout.overriden) + if (!fileTransferSettings.connectTimeout.overridden) fileTransferSettings.connectTimeout = 1; } diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md index 1663b847b..f9fdcbc57 100644 --- a/src/nix/store-prefetch-file.md +++ b/src/nix/store-prefetch-file.md @@ -27,6 +27,6 @@ the resulting store path and the cryptographic hash of the contents of the file. The name component of the store path defaults to the last component of -*url*, but this can be overriden using `--name`. +*url*, but this can be overridden using `--name`. )"" From dd77f71afe6733e9790dd001125c423cb648b7ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Mar 2021 17:10:15 +0100 Subject: [PATCH 459/477] LocalBinaryCacheStore::upsertFile(): Fix race When multiple threads try to upsert the same file, this could fail. Fixes #4667. --- src/libstore/local-binary-cache-store.cc | 5 ++++- tests/ca/substitute.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a58b7733f..964c4017e 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,8 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" +#include + namespace nix { struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig @@ -50,7 +52,8 @@ protected: const std::string & mimeType) override { auto path2 = binaryCacheDir + "/" + path; - Path tmp = path2 + ".tmp." + std::to_string(getpid()); + static std::atomic counter{0}; + Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index 79a6ef8b1..b44fe499a 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -6,6 +6,8 @@ source common.sh sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf +rm -rf $TEST_ROOT/binary_cache + export REMOTE_STORE=file://$TEST_ROOT/binary_cache buildDrvs () { @@ -13,6 +15,7 @@ buildDrvs () { } # Populate the remote cache +clearStore buildDrvs --post-build-hook ../push-to-store.sh # Restart the build on an empty store, ensuring that we don't build From ce791535f63502215d3d41b6ca8d9e62c5fb72e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Mar 2021 14:54:05 +0200 Subject: [PATCH 460/477] nixpkgs/master compatibility --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e59ec9a35..925017472 100644 --- a/flake.nix +++ b/flake.nix @@ -90,7 +90,7 @@ lowdown gmock ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isx86_64 libcpuid; From edd606ae62e213c2a30ff76b8eea4f75ea703d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Sat, 27 Mar 2021 14:15:28 +0100 Subject: [PATCH 461/477] fetchGit: don't prefix "refs/heads/" on ref = "HEAD" This fixes builtins.fetchGit { url = ...; ref = "HEAD"; }, that works in stable nix (v2.3.10), but is broken in nix master: $ ./result/bin/nix repl Welcome to Nix version 2.4pre19700101_dd77f71. Type :? for help. nix-repl> builtins.fetchGit { url = "https://github.com/NixOS/nix"; ref = "HEAD"; } fetching Git repository 'https://github.com/NixOS/nix'fatal: couldn't find remote ref refs/heads/HEAD error: program 'git' failed with exit code 128 The documentation for builtins.fetchGit says ref = "HEAD" is the default, so it should also be supported to explicitly pass it. I came across this issue because poetry2nix can use ref = "HEAD" in some situations. Fixes #4674. --- src/libfetchers/git.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4f9db1bcd..b9a240b13 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -365,7 +365,9 @@ struct GitInputScheme : InputScheme ? "refs/*" : ref->compare(0, 5, "refs/") == 0 ? *ref - : "refs/heads/" + *ref; + : ref == "HEAD" + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; From f2a799b16d193a651f682da3ad2103c20ac82d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 30 Mar 2021 11:39:37 +0200 Subject: [PATCH 462/477] tests: check that builtins.fetchGit { ..., ref = "HEAD"; } works --- tests/fetchGit.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 1e8963d76..88744ee7f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -179,3 +179,13 @@ git clone --depth 1 file://$repo $TEST_ROOT/shallow path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") [[ $path3 = $path6 ]] [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] + +# Explicit ref = "HEAD" should work, and produce the same outPath as without ref +path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") +path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") +[[ $path7 = $path8 ]] + +# ref = "HEAD" should fetch the HEAD revision +rev4=$(git -C $repo rev-parse HEAD) +rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev") +[[ $rev4 = $rev4_nix ]] From 5926200db09ca4d0c5769edf24a3cf2e9f472d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?= <51823984+cgohla@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:54:09 +0100 Subject: [PATCH 463/477] [prerequisites]: add JSON lib dependency --- doc/manual/src/installation/prerequisites-source.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 6825af707..12758c5e1 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -69,3 +69,6 @@ `--disable-seccomp-sandboxing` option to the `configure` script (Not recommended unless your system doesn't support `libseccomp`). To get the library, visit . + + - Niels Lohmann's [JSON library](https://github.com/nlohmann/json). + From 00f00a995458776e33fdda692abe2099196ac566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 Apr 2021 21:32:09 +0200 Subject: [PATCH 464/477] bump actions --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2531a7d35..33035ca1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,9 +13,9 @@ jobs: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v12 + - uses: cachix/install-nix-action@v13 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v8 + - uses: cachix/cachix-action@v9 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -44,8 +44,8 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v12 - - uses: cachix/cachix-action@v8 + - uses: cachix/install-nix-action@v13 + - uses: cachix/cachix-action@v9 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -62,7 +62,7 @@ jobs: steps: - uses: actions/checkout@v2.3.4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@master + - uses: cachix/install-nix-action@v13 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" From f7d9f7c3381acef38e4db2bb2f9e0287c289be54 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Mar 2021 05:48:01 +0000 Subject: [PATCH 465/477] Pull out Buildable into its own file/header in libnixstore --- src/libcmd/installables.cc | 25 ------------------------- src/libcmd/installables.hh | 22 +--------------------- src/libstore/buildable.cc | 33 +++++++++++++++++++++++++++++++++ src/libstore/buildable.hh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 46 deletions(-) create mode 100644 src/libstore/buildable.cc create mode 100644 src/libstore/buildable.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 898e642a5..ca416b9ee 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -20,31 +20,6 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { - nlohmann::json res; - res["path"] = store->printStorePath(path); - return res; -} - -nlohmann::json BuildableFromDrv::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; - } - return res; -} - -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { - auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable); - } - return res; -} - void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b714f097b..d31afd3d5 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,13 +2,12 @@ #include "util.hh" #include "path.hh" +#include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" #include -#include - namespace nix { struct DrvInfo; @@ -16,25 +15,6 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct BuildableOpaque { - StorePath path; - nlohmann::json toJSON(ref store) const; -}; - -struct BuildableFromDrv { - StorePath drvPath; - std::map> outputs; - nlohmann::json toJSON(ref store) const; -}; - -typedef std::variant< - BuildableOpaque, - BuildableFromDrv -> Buildable; - -typedef std::vector Buildables; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); - struct App { std::vector context; diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc new file mode 100644 index 000000000..5cba45b1d --- /dev/null +++ b/src/libstore/buildable.cc @@ -0,0 +1,33 @@ +#include "buildable.hh" +#include "store-api.hh" + +#include + +namespace nix { + +nlohmann::json BuildableOpaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuildableFromDrv::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = path ? store->printStorePath(*path) : ""; + } + return res; +} + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const Buildable & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable); + } + return res; +} + +} diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh new file mode 100644 index 000000000..6177237be --- /dev/null +++ b/src/libstore/buildable.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "util.hh" +#include "path.hh" + +#include + +#include + +namespace nix { + +class Store; + +struct BuildableOpaque { + StorePath path; + nlohmann::json toJSON(ref store) const; +}; + +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + nlohmann::json toJSON(ref store) const; +}; + +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + +typedef std::vector Buildables; + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); + +} From 7a2b566dc8f0f94fdd6acbce90e47cd967f9f134 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 00:47:00 +0000 Subject: [PATCH 466/477] Move `StorePathWithOutputs` into its own header/file In the following commits it will become less prevalent. --- src/libcmd/installables.hh | 1 + src/libstore/derivations.cc | 8 ------- src/libstore/path-with-outputs.cc | 36 +++++++++++++++++++++++++++++++ src/libstore/path-with-outputs.hh | 17 +++++++++++++++ src/libstore/path.cc | 15 ------------- src/libstore/path.hh | 10 --------- src/libstore/store-api.cc | 7 ------ src/libstore/store-api.hh | 1 + 8 files changed, 55 insertions(+), 40 deletions(-) create mode 100644 src/libstore/path-with-outputs.cc create mode 100644 src/libstore/path-with-outputs.hh diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d31afd3d5..e5c6fe208 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe98182bb..f6defd98f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -590,14 +590,6 @@ std::map staticOutputHashes(Store& store, const Derivation& d } -std::string StorePathWithOutputs::to_string(const Store & store) const -{ - return outputs.empty() - ? store.printStorePath(path) - : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); -} - - bool wantOutput(const string & output, const std::set & wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc new file mode 100644 index 000000000..ba15df0a9 --- /dev/null +++ b/src/libstore/path-with-outputs.cc @@ -0,0 +1,36 @@ +#include "store-api.hh" + +namespace nix { + +std::string StorePathWithOutputs::to_string(const Store & store) const +{ + return outputs.empty() + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); +} + + +std::pair parsePathWithOutputs(std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? std::make_pair(s, std::set()) + : std::make_pair(((std::string_view) s).substr(0, n), + tokenizeString>(((std::string_view) s).substr(n + 1), ",")); +} + + +StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +{ + auto [path, outputs] = nix::parsePathWithOutputs(s); + return {parseStorePath(path), std::move(outputs)}; +} + + +StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +{ + auto [path2, outputs] = nix::parsePathWithOutputs(path); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; +} + +} diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh new file mode 100644 index 000000000..a9e3fc7c2 --- /dev/null +++ b/src/libstore/path-with-outputs.hh @@ -0,0 +1,17 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct StorePathWithOutputs +{ + StorePath path; + std::set outputs; + + std::string to_string(const Store & store) const; +}; + +std::pair parsePathWithOutputs(std::string_view s); + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index dc9dc3897..e642abcd5 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -std::pair parsePathWithOutputs(std::string_view s) -{ - size_t n = s.find("!"); - return n == s.npos - ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); -} - -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) -{ - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; -} - } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..06ba0663b 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -69,16 +69,6 @@ typedef std::map> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -struct StorePathWithOutputs -{ - StorePath path; - std::set outputs; - - std::string to_string(const Store & store) const; -}; - -std::pair parsePathWithOutputs(std::string_view s); - } namespace std { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5e321cedf..e3500872c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -53,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const -{ - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; -} - - /* Store paths have the following form: = /- diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5d19e8949..7adbe3b17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "realisation.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 32f4454b9fa3ac30d58e738ece322eb19a0728ba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 01:06:08 +0000 Subject: [PATCH 467/477] Move `StorePathWithOutput` utilities out of store class These are by no means part of the notion of a store, but rather are things that happen to use stores. (Or put another way, there's no way we'd make them virtual methods any time soon.) It's better to move them out of that too-big class then. Also, this helps us remove StorePathWithOutputs from the Store interface altogether next commit. --- src/libexpr/get-drvs.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/path-with-outputs.cc | 12 ++++++------ src/libstore/path-with-outputs.hh | 9 +++++++++ src/libstore/store-api.hh | 7 ------- src/nix-store/nix-store.cc | 4 ++-- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1a3990ea1..7793f26ff 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); this->drvPath = store->printStorePath(drvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f28ab6438..48706bff8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildPaths: { std::vector drvs; for (auto & s : readStrings(from)) - drvs.push_back(store->parsePathWithOutputs(s)); + drvs.push_back(parsePathWithOutputs(*store, s)); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -861,7 +861,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryMissing: { std::vector targets; for (auto & s : readStrings(from)) - targets.push_back(store->parsePathWithOutputs(s)); + targets.push_back(parsePathWithOutputs(*store, s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index ba15df0a9..a898ad09c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -20,17 +20,17 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index a9e3fc7c2..0e34b5aa1 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,4 +14,13 @@ struct StorePathWithOutputs std::pair parsePathWithOutputs(std::string_view s); +class Store; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7adbe3b17..da7ac4460 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -262,11 +262,6 @@ public: PathSet printStorePathSet(const StorePathSet & path) const; - /* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ - StorePathWithOutputs parsePathWithOutputs(const string & s); - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ std::string showPaths(const StorePathSet & paths); @@ -290,8 +285,6 @@ public: result. */ StorePath followLinksToStorePath(std::string_view path) const; - StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; - /* Constructs a unique store path name. */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b684feccb..bfd1299fc 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs) std::vector paths; for (auto & i : opArgs) - paths.push_back(store->followLinksToStorePathWithOutputs(i)); + paths.push_back(followLinksToStorePathWithOutputs(*store, i)); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; @@ -873,7 +873,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths; for (auto & s : readStrings(in)) - paths.push_back(store->parsePathWithOutputs(s)); + paths.push_back(parsePathWithOutputs(*store, s)); getBuildSettings(); From 255d145ba7ac907d1cba8d088da556b591627756 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 03:50:41 +0000 Subject: [PATCH 468/477] Use `BuildableReq` for `buildPaths` and `ensurePath` This avoids an ambiguity where the `StorePathWithOutputs { drvPath, {} }` could mean "build `brvPath`" or "substitute `drvPath`" depending on context. It also brings the internals closer in line to the new CLI, by generalizing the `Buildable` type is used there and makes that distinction already. In doing so, relegate `StorePathWithOutputs` to being a type just for backwards compatibility (CLI and RPC). --- src/libcmd/installables.cc | 7 +-- src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 12 +++-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 3 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 16 +++--- src/libstore/build/local-derivation-goal.cc | 52 +++++++++++++------ src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 47 +++++++++++++++++ src/libstore/buildable.hh | 29 +++++++++-- src/libstore/daemon.cc | 21 +++++--- src/libstore/legacy-ssh-store.cc | 16 ++++-- src/libstore/misc.cc | 49 +++++++++--------- src/libstore/path-with-outputs.cc | 35 +++++++++++++ src/libstore/path-with-outputs.hh | 9 ++++ src/libstore/remote-store.cc | 57 +++++++++++++++++---- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 6 +-- src/libstore/worker-protocol.hh | 22 ++++++++ src/nix-build/nix-build.cc | 4 +- src/nix-env/nix-env.cc | 30 ++++++----- src/nix-env/user-env.cc | 9 +++- src/nix-store/nix-store.cc | 11 ++-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 3 +- src/nix/flake.cc | 5 +- src/nix/profile.cc | 15 +++--- src/nix/run.cc | 2 +- 31 files changed, 364 insertions(+), 126 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ca416b9ee..b68c5f6a7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -679,19 +679,20 @@ Buildables build(ref store, Realise mode, Buildables buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { for (auto & b : i->toBuildables()) { std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path}); + pathsToBuild.push_back(bo); }, [&](BuildableFromDrv bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); - pathsToBuild.push_back({bfd.drvPath, outputNames}); + pathsToBuild.push_back( + BuildableReqFromDrv{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7793f26ff..f774e6493 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "eval-inline.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include #include diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d1afa768..24bc34b74 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); + drvs.push_back({ctx, {outputName}}); } } @@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context) if (!evalSettings.enableImportFromDerivation) throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", - store->printStorePath(drvs.begin()->path)); + store->printStorePath(drvs.begin()->drvPath)); /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed paths. */ diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5baaff3e9..20027e099 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index edc7b5efa..18e0fb57d 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,6 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" +#include "buildable.hh" #include @@ -42,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e7be517e..8680d0bce 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 686364440..d1973d78b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,16 +6,20 @@ namespace nix { -void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & br : reqs) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](BuildableOpaque bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ef43c225..c245527c9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } + +static StorePath pathPartOfReq(const BuildableReq & req) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) { + return bo.path; + }, + [&](BuildableReqFromDrv bfd) { + return bfd.drvPath; + }, + }, req); +} + + +bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +{ + return this->isAllowed(pathPartOfReq(req)); +} + + struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + for (auto & req : paths) { + if (!goal.isAllowed(req)) + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); } next->buildPaths(paths, buildMode); for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); + auto p = std::get_if(&path); + if (!p) continue; + auto & bfd = *p; + auto outputs = next->queryDerivationOutputMap(bfd.drvPath); + for (auto & [outputName, outputPath] : outputs) + if (wantOutput(outputName, bfd.outputs)) + newPaths.insert(outputPath); } StorePathSet closure; @@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); + std::vector allowed; + for (auto & req : targets) { + if (goal.isAllowed(req)) + allowed.emplace_back(req); else - unknown.insert(path.path); + unknown.insert(pathPartOfReq(req)); } next->queryMissing(allowed, willBuild, willSubstitute, diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 47b818a8b..edb93f84e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const BuildableReq & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 616b17e61..fef4cb0cb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->storePath}); + topPaths.push_back(BuildableOpaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 5cba45b1d..63ca1779e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,6 +11,7 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } +template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -30,4 +31,50 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) return res; } + +std::string BuildableOpaque::to_string(const Store & store) const { + return store.printStorePath(path); +} + +template<> +std::string BuildableReqFromDrv::to_string(const Store & store) const { + return store.printStorePath(drvPath) + + "!" + + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); +} + +std::string to_string(const Store & store, const BuildableReq & req) +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + req); +} + + +BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +{ + return {store.parseStorePath(s)}; +} + +template<> +BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + assert(n != s.npos); + auto drvPath = store.parseStorePath(s.substr(0, n)); + auto outputsS = s.substr(n + 1); + std::set outputs; + if (outputsS != "*") + outputs = tokenizeString>(outputsS); + return {drvPath, outputs}; +} + +BuildableReq parseBuildableReq(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? (BuildableReq) BuildableOpaque::parse(store, s) + : (BuildableReq) BuildableReqFromDrv::parse(store, s); +} + } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 6177237be..db78316bd 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path.hh" #include @@ -13,19 +14,37 @@ class Store; struct BuildableOpaque { StorePath path; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableOpaque parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +template +struct BuildableForFromDrv { StorePath drvPath; - std::map> outputs; + Outputs outputs; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableForFromDrv parse(const Store & store, std::string_view); }; -typedef std::variant< +template +using BuildableFor = std::variant< BuildableOpaque, - BuildableFromDrv -> Buildable; + BuildableForFromDrv +>; + +typedef BuildableForFromDrv> BuildableReqFromDrv; +typedef BuildableFor> BuildableReq; + +std::string to_string(const Store & store, const BuildableReq &); + +BuildableReq parseBuildableReq(const Store & store, std::string_view); + +typedef BuildableForFromDrv>> BuildableFromDrv; +typedef BuildableFor>> Buildable; typedef std::vector Buildables; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48706bff8..6b527dcb2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,6 +2,7 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -259,6 +260,18 @@ static void writeValidPathInfo( } } +static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +{ + std::vector reqs; + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + reqs = worker_proto::read(store, from, Phantom> {}); + } else { + for (auto & s : readStrings(from)) + reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + } + return reqs; +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - std::vector drvs; - for (auto & s : readStrings(from)) - drvs.push_back(parsePathWithOutputs(*store, s)); + auto drvs = readBuildableReqs(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -859,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - std::vector targets; - for (auto & s : readStrings(from)) - targets.push_back(parsePathWithOutputs(*store, s)); + auto targets = readBuildableReqs(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a9f53bad9..1cb977be6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "serve-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" @@ -266,14 +267,23 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(*this)); + }, + [&](StorePath drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + }, sOrDrvPath); + } conn->to << ss; putBuildSettings(*conn); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f58816ad8..e702a4f9e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); + pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,24 +177,25 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); + pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); } } } }; - doPath = [&](const StorePathWithOutputs & path) { + doPath = [&](const BuildableReq & req) { { auto state(state_.lock()); - if (!state->done.insert(path.to_string(*this)).second) return; + if (!state->done.insert(to_string(*this, req)).second) return; } - if (path.path.isDerivation()) { - if (!isValidPath(path.path)) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bfd.drvPath); return; } @@ -202,52 +203,54 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; } - if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(StorePath(path.path), *drv); + auto drv = make_ref(derivationFromPath(bfd.drvPath)); + ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); } else - mustBuildDrv(path.path, *drv); + mustBuildDrv(bfd.drvPath, *drv); - } else { + }, + [&](BuildableOpaque bo) { - if (isValidPath(path.path)) return; + if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bo.path); return; } - auto info = infos.find(path.path); + auto info = infos.find(bo.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path.path); + state->willSubstitute.insert(bo.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); - } + pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + }, + }, req); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index a898ad09c..353286ac6 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,3 +1,4 @@ +#include "path-with-outputs.hh" #include "store-api.hh" namespace nix { @@ -10,6 +11,40 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } +BuildableReq StorePathWithOutputs::toBuildableReq() const +{ + if (!outputs.empty() || path.isDerivation()) + return BuildableReqFromDrv { path, outputs }; + else + return BuildableOpaque { path }; +} + + +std::vector toBuildableReqs(const std::vector ss) +{ + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + return reqs; +} + + +std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) -> std::variant { + if (bo.path.isDerivation()) { + // drv path gets interpreted as "build", not "get drv file itself" + return bo.path; + } + return StorePathWithOutputs { bo.path }; + }, + [&](BuildableReqFromDrv bfd) -> std::variant { + return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + }, + }, p); +} + + std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0e34b5aa1..870cac08e 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,6 +1,9 @@ #pragma once +#include + #include "path.hh" +#include "buildable.hh" namespace nix { @@ -10,8 +13,14 @@ struct StorePathWithOutputs std::set outputs; std::string to_string(const Store & store) const; + + BuildableReq toBuildableReq() const; + + static std::variant tryFromBuildableReq(const BuildableReq &); }; +std::vector toBuildableReqs(const std::vector); + std::pair parsePathWithOutputs(std::string_view s); class Store; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ccf095dc2..de1c95ed6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "path-with-outputs.hh" #include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" @@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } + +BuildableReq read(const Store & store, Source & from, Phantom _) +{ + auto s = readString(from); + return parseBuildableReq(store, s); +} + +void write(const Store & store, Sink & out, const BuildableReq & req) +{ + out << to_string(store, req); +} + + Realisation read(const Store & store, Source & from, Phantom _) { std::string rawInput = readString(from); @@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom _) "remote-protocol" ); } + void write(const Store & store, Sink & out, const Realisation & realisation) -{ out << realisation.toJSON().dump(); } +{ + out << realisation.toJSON().dump(); +} + DrvOutput read(const Store & store, Source & from, Phantom _) { @@ -652,16 +670,36 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } +static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +{ + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + worker_proto::write(store, conn->to, reqs); + } else { + Strings ss; + for (auto & p : reqs) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(store)); + }, + [&](StorePath drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn->daemonVersion), + GET_PROTOCOL_MINOR(conn->daemonVersion)); + }, + }, sOrDrvPath); + } + conn->to << ss; + } +} -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -800,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -811,10 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b3a9910a3..20d366038 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e3500872c..8b60bdc62 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back({path}); + paths2.push_back(BuildableOpaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da7ac4460..59d0983df 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "buildable.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index be071dd78..0255726ac 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,9 +86,11 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, BuildableReq); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); #define X_ template @@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); +template +std::vector read(const Store & store, Source & from, Phantom> _) +{ + std::vector resSet; + auto size = readNum(from); + while (size--) { + resSet.push_back(read(store, from, Phantom {})); + } + return resSet; +} + +template +void write(const Store & store, Sink & out, const std::vector & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + template std::set read(const Store & store, Source & from, Phantom> _) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 65b85b304..6f8a61261 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -12,6 +12,7 @@ #include "affinity.hh" #include "util.hh" #include "shared.hh" +#include "path-with-outputs.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" @@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector & paths) { + auto buildPaths = [&](const std::vector & paths0) { + auto paths = toBuildableReqs(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0f10a4cbb..af1c69b87 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -6,6 +6,7 @@ #include "globals.hh" #include "names.hh" #include "profiles.hh" +#include "path-with-outputs.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back({state.store->parseStorePath(drvPath)}); + targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); else - targets.push_back({state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - if (drv.queryDrvPath() != "") { - std::vector paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; - printMissing(globals.state->store, paths); - if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } else { - printMissing(globals.state->store, - {{globals.state->store->parseStorePath(drv.queryOutPath())}}); - if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); - } + std::vector paths { + (drv.queryDrvPath() != "") + ? (BuildableReq) (BuildableReqFromDrv { + globals.state->store->parseStorePath(drv.queryDrvPath()) + }) + : (BuildableReq) (BuildableOpaque { + globals.state->store->parseStorePath(drv.queryOutPath()) + }), + }; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); debug(format("switching to new user environment")); Path generation = createGeneration( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 168ac492b..0ccf960fb 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" @@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); - state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(drvsToBuild), + state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; @@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(topLevelDrvs), + state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index bfd1299fc..21c1e547b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -10,6 +10,7 @@ #include "worker-protocol.hh" #include "graphml.hh" #include "legacy.hh" +#include "path-with-outputs.hh" #include #include @@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path}); + if (build) store->buildPaths({path.toBuildableReq()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -132,7 +133,9 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->queryMissing( + toBuildableReqs(paths), + willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { std::vector paths2; @@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(paths, buildMode); + store->buildPaths(toBuildableReqs(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -879,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(paths); + store->buildPaths(toBuildableReqs(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 48f4eb6e3..e86fbb3f7 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(app.context); + store->buildPaths(toBuildableReqs(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({{drvPath}}); + store->buildPaths({ BuildableReqFromDrv { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d0b140570..616e2073e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" @@ -159,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({{shellDrvPath}}); + store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a2b6c0303..9d6d22a43 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" +#include "path-with-outputs.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -292,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -461,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back({drvPath}); + drvPaths.push_back(BuildableReqFromDrv{drvPath}); } } } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4d275f577..b96e71844 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -260,12 +260,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path, {}}); + pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, [&](BuildableFromDrv bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back({bfd.drvPath, {output.first}}); + pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -388,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -423,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index ec9388234..2e9bb41cc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(app.context); + state->store->buildPaths(toBuildableReqs(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 4fe41c6db390c0295d20f6365ebedaec8ec79e1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:15:25 -0400 Subject: [PATCH 469/477] No templates for `Buildable` and `BuildableReq` --- src/libstore/buildable.cc | 3 --- src/libstore/buildable.hh | 30 +++++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 63ca1779e..7892b94e4 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,6 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } -template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -36,7 +35,6 @@ std::string BuildableOpaque::to_string(const Store & store) const { return store.printStorePath(path); } -template<> std::string BuildableReqFromDrv::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" @@ -56,7 +54,6 @@ BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) return {store.parseStorePath(s)}; } -template<> BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index db78316bd..54e627271 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -20,31 +20,35 @@ struct BuildableOpaque { static BuildableOpaque parse(const Store & store, std::string_view); }; -template -struct BuildableForFromDrv { +struct BuildableReqFromDrv { StorePath drvPath; - Outputs outputs; + std::set outputs; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableForFromDrv parse(const Store & store, std::string_view); + static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -template -using BuildableFor = std::variant< +using BuildableReq = std::variant< BuildableOpaque, - BuildableForFromDrv + BuildableReqFromDrv >; -typedef BuildableForFromDrv> BuildableReqFromDrv; -typedef BuildableFor> BuildableReq; - std::string to_string(const Store & store, const BuildableReq &); BuildableReq parseBuildableReq(const Store & store, std::string_view); -typedef BuildableForFromDrv>> BuildableFromDrv; -typedef BuildableFor>> Buildable; +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + + nlohmann::json toJSON(ref store) const; + static BuildableFromDrv parse(const Store & store, std::string_view); +}; + +using Buildable = std::variant< + BuildableOpaque, + BuildableFromDrv +>; typedef std::vector Buildables; From 9dfb97c987d8b9d6a3d15f016e40f22f91deb764 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:24:42 -0400 Subject: [PATCH 470/477] "newtype" BuildableReq This makes for better types errors and allows us to give it methods. --- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/libstore/buildable.cc | 6 +++--- src/libstore/buildable.hh | 14 +++++++++++--- src/libstore/misc.cc | 4 ++-- src/libstore/path-with-outputs.cc | 2 +- src/libstore/remote-store.cc | 4 ++-- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8680d0bce..8396abbcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); + BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); + BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index d1973d78b..fc6294545 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -19,7 +19,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMo [&](BuildableOpaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, - }, br); + }, br.raw()); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c245527c9..6cc384719 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1200,7 +1200,7 @@ static StorePath pathPartOfReq(const BuildableReq & req) [&](BuildableReqFromDrv bfd) { return bfd.drvPath; }, - }, req); + }, req.raw()); } @@ -1340,7 +1340,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo for (auto & req : paths) { if (!goal.isAllowed(req)) - throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } next->buildPaths(paths, buildMode); diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 7892b94e4..31fef2faa 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -41,11 +41,11 @@ std::string BuildableReqFromDrv::to_string(const Store & store) const { + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string to_string(const Store & store, const BuildableReq & req) +std::string BuildableReq::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, - req); + this->raw()); } @@ -66,7 +66,7 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq parseBuildableReq(const Store & store, std::string_view s) +BuildableReq BuildableReq::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 54e627271..8317f3995 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -28,14 +28,22 @@ struct BuildableReqFromDrv { static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -using BuildableReq = std::variant< +using _BuildableReqRaw = std::variant< BuildableOpaque, BuildableReqFromDrv >; -std::string to_string(const Store & store, const BuildableReq &); +struct BuildableReq : _BuildableReqRaw { + using Raw = _BuildableReqRaw; + using Raw::Raw; -BuildableReq parseBuildableReq(const Store & store, std::string_view); + inline const Raw & raw() const { + return static_cast(*this); + } + + std::string to_string(const Store & store) const; + static BuildableReq parse(const Store & store, std::string_view); +}; struct BuildableFromDrv { StorePath drvPath; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e702a4f9e..abfae1502 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -187,7 +187,7 @@ void Store::queryMissing(const std::vector & targets, { auto state(state_.lock()); - if (!state->done.insert(to_string(*this, req)).second) return; + if (!state->done.insert(req.to_string(*this)).second) return; } std::visit(overloaded { @@ -250,7 +250,7 @@ void Store::queryMissing(const std::vector & targets, for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); }, - }, req); + }, req.raw()); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 353286ac6..2898b8d4f 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -41,7 +41,7 @@ std::variant StorePathWithOutputs::tryFromBuild [&](BuildableReqFromDrv bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, - }, p); + }, p.raw()); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de1c95ed6..cb6402213 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -55,12 +55,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) BuildableReq read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return parseBuildableReq(store, s); + return BuildableReq::parse(store, s); } void write(const Store & store, Sink & out, const BuildableReq & req) { - out << to_string(store, req); + out << req.to_string(store); } From 9b805d36ac70545fc4c0d863e21e0c2e5f2518a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:48:18 -0400 Subject: [PATCH 471/477] Rename Buildable --- src/libcmd/command.cc | 6 +-- src/libcmd/command.hh | 4 +- src/libcmd/installables.cc | 44 ++++++++++----------- src/libcmd/installables.hh | 6 +-- src/libexpr/primops.cc | 6 +-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 2 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 6 +-- src/libstore/build/local-derivation-goal.cc | 16 ++++---- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 24 +++++------ src/libstore/buildable.hh | 37 +++++++++-------- src/libstore/daemon.cc | 12 +++--- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/misc.cc | 16 ++++---- src/libstore/path-with-outputs.cc | 18 ++++----- src/libstore/path-with-outputs.hh | 6 +-- src/libstore/remote-store.cc | 18 ++++----- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/worker-protocol.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 12 +++--- src/nix-env/user-env.cc | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/build.cc | 6 +-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 8 ++-- src/nix/flake.cc | 4 +- src/nix/log.cc | 6 +-- src/nix/profile.cc | 14 +++---- src/nix/run.cc | 2 +- 35 files changed, 165 insertions(+), 162 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d29954f67..dc1fbc43f 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -162,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const Buildables & buildables) +void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) { if (!profile) return; @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { result.push_back(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index e66c697eb..9e18c6e51 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, @@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const Buildables & buildables); + void updateProfile(const DerivedPathsWithHints & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b68c5f6a7..f091ac186 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -285,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -Buildable Installable::toBuildable() +DerivedPathWithHints Installable::toDerivedPathWithHints() { - auto buildables = toBuildables(); + auto buildables = toDerivedPathsWithHints(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -321,7 +321,7 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - Buildables toBuildables() override + DerivedPathsWithHints toDerivedPathsWithHints() override { if (storePath.isDerivation()) { std::map> outputs; @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - BuildableFromDrv { + DerivedPathWithHintsBuilt { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - BuildableOpaque { + DerivedPathOpaque { .path = storePath, } }; @@ -349,9 +349,9 @@ struct InstallableStorePath : Installable } }; -Buildables InstallableValue::toBuildables() +DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() { - Buildables res; + DerivedPathsWithHints res; std::map>> drvsToOutputs; @@ -364,7 +364,7 @@ Buildables InstallableValue::toBuildables() } for (auto & i : drvsToOutputs) - res.push_back(BuildableFromDrv { i.first, i.second }); + res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); return res; } @@ -671,28 +671,28 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - Buildables buildables; + DerivedPathsWithHints buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toBuildables()) { + for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( - BuildableReqFromDrv{bfd.drvPath, outputNames}); + DerivedPath::Built{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { res.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -751,8 +751,8 @@ std::set toRealisedPaths( settings.readOnlyMode = true; for (auto & i : installables) - for (auto & b : i->toBuildables()) - if (auto bfd = std::get_if(&b)) + for (auto & b : i->toDerivedPathsWithHints()) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -787,9 +787,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) + for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,7 +798,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { drvPaths.insert(bfd.drvPath); }, }, b); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e5c6fe208..0bc932b52 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -29,9 +29,9 @@ struct Installable virtual std::string what() = 0; - virtual Buildables toBuildables() = 0; + virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0; - Buildable toBuildable(); + DerivedPathWithHints toDerivedPathWithHints(); App toApp(EvalState & state); @@ -74,7 +74,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - Buildables toBuildables() override; + DerivedPathsWithHints toDerivedPathsWithHints() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 24bc34b74..428adf4c2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -56,8 +56,8 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - std::vector buildReqs; - for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(buildReqs); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 20027e099..09af57871 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 18e0fb57d..9cb9e6da2 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -43,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8396abbcd..3ce538f77 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index fc6294545..732d4785d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,17 +6,17 @@ namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; for (auto & br : reqs) { std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, }, br.raw()); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6cc384719..7c1402918 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1191,20 +1191,20 @@ void LocalDerivationGoal::writeStructuredAttrs() } -static StorePath pathPartOfReq(const BuildableReq & req) +static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { return bo.path; }, - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { return bfd.drvPath; }, }, req.raw()); } -bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +bool LocalDerivationGoal::isAllowed(const DerivedPath & req) { return this->isAllowed(pathPartOfReq(req)); } @@ -1332,7 +1332,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1346,7 +1346,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo next->buildPaths(paths, buildMode); for (auto & path : paths) { - auto p = std::get_if(&path); + auto p = std::get_if(&path); if (!p) continue; auto & bfd = *p; auto outputs = next->queryDerivationOutputMap(bfd.drvPath); @@ -1380,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1388,7 +1388,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; + std::vector allowed; for (auto & req : targets) { if (goal.isAllowed(req)) allowed.emplace_back(req); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index edb93f84e..d30be2351 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,7 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } - bool isAllowed(const BuildableReq & req); + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index fef4cb0cb..6c04d3ed3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableOpaque{goal->storePath}); + topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 31fef2faa..a8c0c70b1 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -5,13 +5,13 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { +nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { nlohmann::json res; res["path"] = store->printStorePath(path); return res; } -nlohmann::json BuildableFromDrv::toJSON(ref store) const { +nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -20,9 +20,9 @@ nlohmann::json BuildableFromDrv::toJSON(ref store) const { return res; } -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store) { auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { + for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); }, buildable); @@ -31,17 +31,17 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) } -std::string BuildableOpaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } -std::string BuildableReqFromDrv::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string BuildableReq::to_string(const Store & store) const +std::string DerivedPath::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, @@ -49,12 +49,12 @@ std::string BuildableReq::to_string(const Store & store) const } -BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) { return {store.parseStorePath(s)}; } -BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); assert(n != s.npos); @@ -66,12 +66,12 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq BuildableReq::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? (BuildableReq) BuildableOpaque::parse(store, s) - : (BuildableReq) BuildableReqFromDrv::parse(store, s); + ? (DerivedPath) DerivedPath::Opaque::parse(store, s) + : (DerivedPath) DerivedPath::Built::parse(store, s); } } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 8317f3995..0a0cf8105 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -12,54 +12,57 @@ namespace nix { class Store; -struct BuildableOpaque { +struct DerivedPathOpaque { StorePath path; nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableOpaque parse(const Store & store, std::string_view); + static DerivedPathOpaque parse(const Store & store, std::string_view); }; -struct BuildableReqFromDrv { +struct DerivedPathBuilt { StorePath drvPath; std::set outputs; std::string to_string(const Store & store) const; - static BuildableReqFromDrv parse(const Store & store, std::string_view); + static DerivedPathBuilt parse(const Store & store, std::string_view); }; -using _BuildableReqRaw = std::variant< - BuildableOpaque, - BuildableReqFromDrv +using _DerivedPathRaw = std::variant< + DerivedPathOpaque, + DerivedPathBuilt >; -struct BuildableReq : _BuildableReqRaw { - using Raw = _BuildableReqRaw; +struct DerivedPath : _DerivedPathRaw { + using Raw = _DerivedPathRaw; using Raw::Raw; + using Opaque = DerivedPathOpaque; + using Built = DerivedPathBuilt; + inline const Raw & raw() const { return static_cast(*this); } std::string to_string(const Store & store) const; - static BuildableReq parse(const Store & store, std::string_view); + static DerivedPath parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; nlohmann::json toJSON(ref store) const; - static BuildableFromDrv parse(const Store & store, std::string_view); + static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using Buildable = std::variant< - BuildableOpaque, - BuildableFromDrv +using DerivedPathWithHints = std::variant< + DerivedPath::Opaque, + DerivedPathWithHintsBuilt >; -typedef std::vector Buildables; +typedef std::vector DerivedPathsWithHints; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6b527dcb2..affd60472 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,14 +260,14 @@ static void writeValidPathInfo( } } -static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { - std::vector reqs; + std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - reqs = worker_proto::read(store, from, Phantom> {}); + reqs = worker_proto::read(store, from, Phantom> {}); } else { for (auto & s : readStrings(from)) - reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); } return reqs; } @@ -506,7 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - auto drvs = readBuildableReqs(*store, clientVersion, from); + auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -870,7 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - auto targets = readBuildableReqs(*store, clientVersion, from); + auto targets = readDerivedPaths(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1cb977be6..edaf75136 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,14 +267,14 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(*this)); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index abfae1502..a99a2fc78 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,13 +177,13 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } )); } } } }; - doPath = [&](const BuildableReq & req) { + doPath = [&](const DerivedPath & req) { { auto state(state_.lock()); @@ -191,7 +191,7 @@ void Store::queryMissing(const std::vector & targets, } std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); @@ -224,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, mustBuildDrv(bfd.drvPath, *drv); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { if (isValidPath(bo.path)) return; @@ -248,7 +248,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); }; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 2898b8d4f..865d64cf2 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -11,34 +11,34 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } -BuildableReq StorePathWithOutputs::toBuildableReq() const +DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty() || path.isDerivation()) - return BuildableReqFromDrv { path, outputs }; + return DerivedPath::Built { path, outputs }; else - return BuildableOpaque { path }; + return DerivedPath::Opaque { path }; } -std::vector toBuildableReqs(const std::vector ss) +std::vector toDerivedPaths(const std::vector ss) { - std::vector reqs; - for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); return reqs; } -std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](BuildableOpaque bo) -> std::variant { + [&](DerivedPath::Opaque bo) -> std::variant { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](BuildableReqFromDrv bfd) -> std::variant { + [&](DerivedPath::Built bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, }, p.raw()); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 870cac08e..749348398 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,12 +14,12 @@ struct StorePathWithOutputs std::string to_string(const Store & store) const; - BuildableReq toBuildableReq() const; + DerivedPath toDerivedPath() const; - static std::variant tryFromBuildableReq(const BuildableReq &); + static std::variant tryFromDerivedPath(const DerivedPath &); }; -std::vector toBuildableReqs(const std::vector); +std::vector toDerivedPaths(const std::vector); std::pair parsePathWithOutputs(std::string_view s); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cb6402213..761b4a087 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -52,13 +52,13 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) } -BuildableReq read(const Store & store, Source & from, Phantom _) +DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return BuildableReq::parse(store, s); + return DerivedPath::parse(store, s); } -void write(const Store & store, Sink & out, const BuildableReq & req) +void write(const Store & store, Sink & out, const DerivedPath & req) { out << req.to_string(store); } @@ -670,14 +670,14 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } -static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { worker_proto::write(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(store)); @@ -694,12 +694,12 @@ static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, con } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeBuildableReqs(*this, conn, drvPaths); + writeDerivedPaths(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -838,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -849,7 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - writeBuildableReqs(*this, conn, targets); + writeDerivedPaths(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 20d366038..6cf76a46d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8b60bdc62..93fcb068f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(BuildableOpaque{path}); + paths2.push_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 59d0983df..483f3c5fa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0255726ac..001ed25e3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,7 +86,7 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, BuildableReq); +MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6f8a61261..d46bc1f2b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -323,7 +323,7 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); auto buildPaths = [&](const std::vector & paths0) { - auto paths = toBuildableReqs(paths0); + auto paths = toDerivedPaths(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index af1c69b87..e04954d45 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -419,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); + targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)}); else - targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -694,12 +694,12 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - std::vector paths { + std::vector paths { (drv.queryDrvPath() != "") - ? (BuildableReq) (BuildableReqFromDrv { + ? (DerivedPath) (DerivedPath::Built { globals.state->store->parseStorePath(drv.queryDrvPath()) }) - : (BuildableReq) (BuildableOpaque { + : (DerivedPath) (DerivedPath::Opaque { globals.state->store->parseStorePath(drv.queryOutPath()) }), }; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 0ccf960fb..5ceb2ae67 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -43,7 +43,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug(format("building user environment dependencies")); state.store->buildPaths( - toBuildableReqs(drvsToBuild), + toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ @@ -140,7 +140,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); state.store->buildPaths( - toBuildableReqs(topLevelDrvs), + toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 21c1e547b..b327793e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -63,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path.toBuildableReq()}); + if (build) store->buildPaths({path.toDerivedPath()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -134,7 +134,7 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing( - toBuildableReqs(paths), + toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { @@ -151,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(toBuildableReqs(paths), buildMode); + store->buildPaths(toDerivedPaths(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -882,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(toBuildableReqs(paths)); + store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/build.cc b/src/nix/build.cc index 724ce9d79..0529ed382 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -80,7 +80,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile updateProfile(buildables); - if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); } }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e86fbb3f7..53dccc63a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(toBuildableReqs(app.context)); + store->buildPaths(toDerivedPaths(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({ BuildableReqFromDrv { drvPath } }); + store->buildPaths({ DerivedPath::Built { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 616e2073e..cae6ded40 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -160,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; @@ -265,7 +265,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toBuildable(); + auto buildable = installable->toDerivedPathWithHints(); auto doRedirect = [&](const StorePath & path) { auto from = store->printStorePath(path); @@ -277,10 +277,10 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const BuildableOpaque & bo) { + [&](const DerivedPathOpaque & bo) { doRedirect(bo.path); }, - [&](const BuildableFromDrv & bfd) { + [&](const DerivedPathWithHintsBuilt & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9d6d22a43..62a413e27 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -293,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -462,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(BuildableReqFromDrv{drvPath}); + drvPaths.push_back(DerivedPath::Built{drvPath}); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index 67d3742d6..5010e3326 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuildable(); + auto b = installable->toDerivedPathWithHints(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { return sub->getBuildLog(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b96e71844..ad824dd70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -259,16 +259,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); + pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -391,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -426,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index 2e9bb41cc..ba60e57d8 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(toBuildableReqs(app.context)); + state->store->buildPaths(toDerivedPaths(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 179582872de60863fcabcf471f98930a25fd6df3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:05:21 -0400 Subject: [PATCH 472/477] Make `DerivedPathWithHints` a newtype This allows us to namespace its constructors under it. --- src/libcmd/command.cc | 6 +++--- src/libcmd/installables.cc | 26 +++++++++++++------------- src/libstore/buildable.cc | 4 ++-- src/libstore/buildable.hh | 15 ++++++++++++++- src/nix/build.cc | 6 +++--- src/nix/develop.cc | 6 +++--- src/nix/log.cc | 6 +++--- src/nix/profile.cc | 6 +++--- 8 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index dc1fbc43f..9da470c15 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { result.push_back(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ @@ -181,7 +181,7 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) result.push_back(*output.second); } }, - }, buildable); + }, buildable.raw()); } if (result.size() != 1) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f091ac186..5d3026c1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - DerivedPathWithHintsBuilt { + DerivedPathWithHints::Built { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - DerivedPathOpaque { + DerivedPathWithHints::Opaque { .path = storePath, } }; @@ -364,7 +364,7 @@ DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() } for (auto & i : drvsToOutputs) - res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); + res.push_back(DerivedPathWithHints::Built { i.first, i.second }); return res; } @@ -684,17 +684,17 @@ DerivedPathsWithHints build(ref store, Realise mode, for (auto & i : installables) { for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( DerivedPath::Built{bfd.drvPath, outputNames}); }, - }, b); + }, b.raw()); buildables.push_back(std::move(b)); } } @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { res.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -745,14 +745,14 @@ std::set toRealisedPaths( } } }, - }, b); + }, b.raw()); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) - if (auto bfd = std::get_if(&b)) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -789,7 +789,7 @@ StorePathSet toDerivations(ref store, for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,10 +798,10 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { drvPaths.insert(bfd.drvPath); }, - }, b); + }, b.raw()); return drvPaths; } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index a8c0c70b1..eee38ba10 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,7 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } -nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { +nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -25,7 +25,7 @@ nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildab for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); - }, buildable); + }, buildable.raw()); } return res; } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 0a0cf8105..ce5ae5fc0 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -56,11 +56,24 @@ struct DerivedPathWithHintsBuilt { static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using DerivedPathWithHints = std::variant< +using _DerivedPathWithHintsRaw = std::variant< DerivedPath::Opaque, DerivedPathWithHintsBuilt >; +struct DerivedPathWithHints : _DerivedPathWithHintsRaw { + using Raw = _DerivedPathWithHintsRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = DerivedPathWithHintsBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + +}; + typedef std::vector DerivedPathsWithHints; nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); diff --git a/src/nix/build.cc b/src/nix/build.cc index 0529ed382..03159b6cc 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -75,7 +75,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildable); + }, buildable.raw()); } updateProfile(buildables); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index cae6ded40..7cc7b85be 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -277,14 +277,14 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const DerivedPathOpaque & bo) { + [&](const DerivedPathWithHints::Opaque & bo) { doRedirect(bo.path); }, - [&](const DerivedPathWithHintsBuilt & bfd) { + [&](const DerivedPathWithHints::Built & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, - }, buildable); + }, buildable.raw()); } return rewriteStrings(script, rewrites); diff --git a/src/nix/log.cc b/src/nix/log.cc index 5010e3326..638bb5073 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -35,13 +35,13 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, - }, b); + }, b.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ad824dd70..667904cd2 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -259,11 +259,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? @@ -272,7 +272,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.storePaths.insert(output.second); } }, - }, buildable); + }, buildable.raw()); manifest.elements.emplace_back(std::move(element)); } From d8fa7517fad4272e20ff9b9b740c91158bc685e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:33:28 -0400 Subject: [PATCH 473/477] buildable.{cc,hh} -> derived-path.{cc,hh} --- src/libcmd/installables.hh | 2 +- src/libmain/shared.hh | 2 +- src/libstore/{buildable.cc => derived-path.cc} | 2 +- src/libstore/{buildable.hh => derived-path.hh} | 0 src/libstore/path-with-outputs.hh | 2 +- src/libstore/store-api.hh | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/libstore/{buildable.cc => derived-path.cc} (98%) rename src/libstore/{buildable.hh => derived-path.hh} (100%) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 0bc932b52..403403c07 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "path.hh" #include "path-with-outputs.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9cb9e6da2..05277d90a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,7 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include diff --git a/src/libstore/buildable.cc b/src/libstore/derived-path.cc similarity index 98% rename from src/libstore/buildable.cc rename to src/libstore/derived-path.cc index eee38ba10..13833c58e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,4 @@ -#include "buildable.hh" +#include "derived-path.hh" #include "store-api.hh" #include diff --git a/src/libstore/buildable.hh b/src/libstore/derived-path.hh similarity index 100% rename from src/libstore/buildable.hh rename to src/libstore/derived-path.hh diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 749348398..4c4023dcb 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -3,7 +3,7 @@ #include #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 483f3c5fa..f66298991 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 125a824228dbac0bb82023953f45318ea93e7ffa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:56:48 -0400 Subject: [PATCH 474/477] Document the derived path types. --- src/libstore/derived-path.hh | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index ce5ae5fc0..7a2fe59de 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -2,7 +2,6 @@ #include "util.hh" #include "path.hh" -#include "path.hh" #include @@ -12,6 +11,13 @@ namespace nix { class Store; +/** + * An opaque derived path. + * + * Opaque derived paths are just store paths, and fully evaluated. They + * cannot be simplified further. Since they are opaque, they cannot be + * built, but they can fetched. + */ struct DerivedPathOpaque { StorePath path; @@ -20,6 +26,18 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const Store & store, std::string_view); }; +/** + * A derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ struct DerivedPathBuilt { StorePath drvPath; std::set outputs; @@ -33,6 +51,16 @@ using _DerivedPathRaw = std::variant< DerivedPathBuilt >; +/** + * A "derived path" is a very simple sort of expression that evaluates + * to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; using Raw::Raw; @@ -48,6 +76,11 @@ struct DerivedPath : _DerivedPathRaw { static DerivedPath parse(const Store & store, std::string_view); }; +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'DerivedPathWithHints' for more an explanation. + */ struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; @@ -61,6 +94,21 @@ using _DerivedPathWithHintsRaw = std::variant< DerivedPathWithHintsBuilt >; +/** + * A derived path with hints in the form of optional concrete output paths in the built case. + * + * This type is currently just used by the CLI. The paths are filled in + * during evaluation for derivations that know what paths they will + * produce in advanced, i.e. input-addressed or fixed-output content + * addressed derivations. + * + * That isn't very good, because it puts floating content-addressed + * derivations "at a disadvantage". It would be better to never rely on + * the output path of unbuilt derivations, and exclusively use the + * realizations types to work with built derivations' concrete output + * paths. + */ +// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. struct DerivedPathWithHints : _DerivedPathWithHintsRaw { using Raw = _DerivedPathWithHintsRaw; using Raw::Raw; From 8a29052cb2f52ef2c82c36fb3818fd0f66349729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Apr 2021 12:21:31 +0200 Subject: [PATCH 475/477] PathSubstitutionGoal: Clean up pipe If there were many top-level goals (which are not destroyed until the very end), commands like $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' \ /run/current-system --no-check-sigs --substitute-on-destination could fail with "Too many open files". So now we do some explicit cleanup from amDone(). It would be cleaner to separate goals from their temporary internal state, but that would be a bigger refactor. --- src/libstore/build/goal.cc | 2 ++ src/libstore/build/goal.hh | 2 ++ src/libstore/build/substitution-goal.cc | 31 +++++++++++++++++-------- src/libstore/build/substitution-goal.hh | 4 +++- src/libstore/build/worker.cc | 1 + src/libutil/util.cc | 17 ++++++++++---- src/libutil/util.hh | 3 ++- 7 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 2dd7a4d37..9de40bdf2 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional ex) } waiters.clear(); worker.removeGoal(shared_from_this()); + + cleanup(); } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index fca4f2d00..e6bf628cb 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this virtual string key() = 0; void amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7b1ac126e..e56cfadbe 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -20,15 +20,7 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & PathSubstitutionGoal::~PathSubstitutionGoal() { - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } + cleanup(); } @@ -63,6 +55,8 @@ void PathSubstitutionGoal::tryNext() { trace("trying next substituter"); + cleanup(); + if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ @@ -205,7 +199,7 @@ void PathSubstitutionGoal::tryToRun() thr = std::thread([this]() { try { /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); + Finally updateStats([this]() { outPipe.writeSide.close(); }); Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); @@ -288,4 +282,21 @@ void PathSubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } + +void PathSubstitutionGoal::cleanup() +{ + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + + outPipe.close(); + } catch (...) { + ignoreException(); + } +} + + } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3b3cb7e32..70c806d23 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -14,7 +14,7 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ + different when the stores have different names. */ std::optional subPath; /* The remaining substituters. */ @@ -79,6 +79,8 @@ public: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; + + void cleanup() override; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c04d3ed3..0f2ade348 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -128,6 +128,7 @@ void Worker::removeGoal(GoalPtr goal) nix::removeGoal(subGoal, drvOutputSubstitutionGoals); else assert(false); + if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals diff --git a/src/libutil/util.cc b/src/libutil/util.cc index dea9c74b7..a6fcabbec 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -752,13 +752,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {} AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} -AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} { that.fd = -1; } -AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that) +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) { close(); fd = that.fd; @@ -789,6 +789,7 @@ void AutoCloseFD::close() if (::close(fd) == -1) /* This should never happen. */ throw SysError("closing file descriptor %1%", fd); + fd = -1; } } @@ -822,6 +823,12 @@ void Pipe::create() } +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + ////////////////////////////////////////////////////////////////////// @@ -1121,7 +1128,7 @@ void runProgram2(const RunOptions & options) throw SysError("executing '%1%'", options.program); }, processOptions); - out.writeSide = -1; + out.writeSide.close(); std::thread writerThread; @@ -1134,7 +1141,7 @@ void runProgram2(const RunOptions & options) if (source) { - in.readSide = -1; + in.readSide.close(); writerThread = std::thread([&]() { try { std::vector buf(8 * 1024); @@ -1151,7 +1158,7 @@ void runProgram2(const RunOptions & options) } catch (...) { promise.set_exception(std::current_exception()); } - in.writeSide = -1; + in.writeSide.close(); }); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ad49c65b3..ef5e5012b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -188,7 +188,6 @@ public: class AutoCloseFD { int fd; - void close(); public: AutoCloseFD(); AutoCloseFD(int fd); @@ -200,6 +199,7 @@ public: int get() const; explicit operator bool() const; int release(); + void close(); }; @@ -216,6 +216,7 @@ class Pipe public: AutoCloseFD readSide, writeSide; void create(); + void close(); }; From 9b9e703df41d75949272059f9b8bc8b763e91fce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Apr 2021 13:10:02 +0200 Subject: [PATCH 476/477] restoreSignals() + restoreAffinity() -> restoreProcessContext() --- src/libmain/shared.cc | 4 ++-- src/libstore/ssh.cc | 4 ++-- src/libutil/util.cc | 14 +++++++++++--- src/libutil/util.hh | 8 +++++--- src/nix-build/nix-build.cc | 4 +--- src/nix/develop.cc | 3 +-- src/nix/edit.cc | 3 ++- src/nix/repl.cc | 30 +++++++----------------------- src/nix/run.cc | 4 +--- 9 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 09af57871..86930c2e3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -310,7 +310,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { - restoreSignals(); + restoreProcessContext(); setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), nullptr); throw SysError("command 'man %1%' failed", name.c_str()); @@ -373,7 +373,7 @@ RunPager::RunPager() throw SysError("dupping stdin"); if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); - restoreSignals(); + restoreProcessContext(); if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr); execlp("pager", "pager", nullptr); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 235eed37a..93f72675d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -50,7 +50,7 @@ std::unique_ptr SSHMaster::startCommand(const std::string options.dieWithParent = false; conn->sshPid = startProcess([&]() { - restoreSignals(); + restoreProcessContext(); close(in.writeSide.get()); close(out.readSide.get()); @@ -110,7 +110,7 @@ Path SSHMaster::startMaster() options.dieWithParent = false; state->sshMaster = startProcess([&]() { - restoreSignals(); + restoreProcessContext(); close(out.readSide.get()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a6fcabbec..06821abb1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1116,7 +1116,7 @@ void runProgram2(const RunOptions & options) Strings args_(options.args); args_.push_front(options.program); - restoreSignals(); + restoreProcessContext(); if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); @@ -1612,12 +1612,19 @@ void startSignalHandlerThread() std::thread(signalHandlerThread, set).detach(); } -void restoreSignals() +static void restoreSignals() { if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } +void restoreProcessContext() +{ + restoreSignals(); + + restoreAffinity(); +} + /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { @@ -1680,10 +1687,11 @@ string showBytes(uint64_t bytes) } +// FIXME: move to libstore/build void commonChildInit(Pipe & logPipe) { const static string pathNullDevice = "/dev/null"; - restoreSignals(); + restoreProcessContext(); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ef5e5012b..4762d1e55 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -300,6 +300,11 @@ std::pair runProgram(const RunOptions & options); void runProgram2(const RunOptions & options); +/* Restore the original inherited Unix process context (such as signal + masks, stack size, CPU affinity). */ +void restoreProcessContext(); + + class ExecError : public Error { public: @@ -513,9 +518,6 @@ class Callback; on the current thread (and thus any threads created by it). */ void startSignalHandlerThread(); -/* Restore default signal handling. */ -void restoreSignals(); - struct InterruptCallback { virtual ~InterruptCallback() { }; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d46bc1f2b..9acbedda2 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -422,8 +422,6 @@ static void main_nix_build(int argc, char * * argv) } else env[var.first] = var.second; - restoreAffinity(); - /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't @@ -473,7 +471,7 @@ static void main_nix_build(int argc, char * * argv) auto argPtrs = stringsToCharPtrs(args); - restoreSignals(); + restoreProcessContext(); logger->stop(); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7cc7b85be..498a7b45c 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -462,8 +462,7 @@ struct CmdDevelop : Common, MixEnvironment auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; - restoreAffinity(); - restoreSignals(); + restoreProcessContext(); execvp(shell.c_str(), stringsToCharPtrs(args).data()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 6472dd27a..b26417b18 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -42,7 +42,8 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); - restoreSignals(); + restoreProcessContext(); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bce8d31dc..eed79c332 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -343,24 +343,6 @@ StringSet NixRepl::completePrefix(string prefix) } -static int runProgram(const string & program, const Strings & args) -{ - Strings args2(args); - args2.push_front(program); - - Pid pid; - pid = fork(); - if (pid == -1) throw SysError("forking"); - if (pid == 0) { - restoreAffinity(); - execvp(program.c_str(), stringsToCharPtrs(args2).data()); - _exit(1); - } - - return pid.wait(); -} - - bool isVarName(const string & s) { if (s.size() == 0) return false; @@ -462,7 +444,7 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, args); + runProgram(editor, true, args); // Reload right after exiting the editor state->resetFileCache(); @@ -481,7 +463,7 @@ bool NixRepl::processLine(string line) state->callFunction(f, v, result, Pos()); StorePath drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)}); + runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { @@ -494,16 +476,18 @@ bool NixRepl::processLine(string line) /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) { + try { + runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw}); auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputsAndOptPaths(*state->store)) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second)); + } catch (ExecError &) { } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw}); + runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw}); + runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw}); } } diff --git a/src/nix/run.cc b/src/nix/run.cc index ba60e57d8..b5d8ab38a 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -31,9 +31,7 @@ struct RunCommon : virtual Command { stopProgressBar(); - restoreSignals(); - - restoreAffinity(); + restoreProcessContext(); /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location From 906adadacd2d1c98346a2f42c0b42a32d2806d94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Apr 2021 13:40:13 +0200 Subject: [PATCH 477/477] Restore stack size in child processes Fixes #4673. --- src/libutil/util.cc | 27 +++++++++++++++++++++++++++ src/libutil/util.hh | 4 ++++ src/nix/main.cc | 13 +------------ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 06821abb1..60b318559 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -32,6 +32,7 @@ #ifdef __linux__ #include +#include #endif @@ -1618,11 +1619,37 @@ static void restoreSignals() throw SysError("restoring signals"); } +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + void restoreProcessContext() { restoreSignals(); restoreAffinity(); + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif } /* RAII helper to automatically deregister a callback. */ diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 4762d1e55..f84d0fb31 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -300,6 +300,10 @@ std::pair runProgram(const RunOptions & options); void runProgram2(const RunOptions & options); +/* Change the stack size. */ +void setStackSize(size_t stackSize); + + /* Restore the original inherited Unix process context (such as signal masks, stack size, CPU affinity). */ void restoreProcessContext(); diff --git a/src/nix/main.cc b/src/nix/main.cc index f8701ee56..008482be3 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -17,10 +17,6 @@ #include #include -#if __linux__ -#include -#endif - #include extern std::string chrootHelperName; @@ -335,14 +331,7 @@ int main(int argc, char * * argv) { // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. - #if __linux__ - rlim_t stackSize = 64 * 1024 * 1024; - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif + nix::setStackSize(64 * 1024 * 1024); return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv);