From 59f2dd8e8da1f82aa9e29e30ba1df643434a9254 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 6 Oct 2020 20:08:51 +0200 Subject: [PATCH 001/113] libfetchers/github: allow slashes in refs Refs #4061 --- src/libfetchers/github.cc | 20 +++++++++++++++++--- src/libutil/url-parts.hh | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..3d1cc15e2 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme std::optional ref; std::optional host_url; - if (path.size() == 2) { - } else if (path.size() == 3) { + auto size = path.size(); + if (size == 3) { if (std::regex_match(path[2], revRegex)) rev = Hash::parseAny(path[2], htSHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else + } else if (size > 3) { + std::string rs; + for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { + rs += *i; + if (std::next(i) != path.end()) { + rs += "/"; + } + } + + if (std::regex_match(rs, refRegex)) { + ref = rs; + } else { + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + } + } else if (size < 2) throw BadURL("URL '%s' is invalid", url.url); for (auto &[name, value] : url.query) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 68be15cb0..e0e2809fd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -22,7 +22,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; // A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check extern std::regex refRegex; // Instead of defining what a good Git Ref is, we define what a bad Git Ref is From 107c91f5fe6248548c292d28d0ad53c0de7ceeba Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 16:48:35 -0700 Subject: [PATCH 002/113] auto-call error --- src/libexpr/eval.cc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6366050c..e52e8dcf2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,7 +1370,28 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + An 'auto-call' is when a nix expression is evaluated without any external arguments. If that + nix expression is a function, and that function's arguments all have default values, then all is well. + + But if the function arguments don't have default values, then evaluation fails. + + The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... + + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + + See this nix pill for more information re callPackage format: + https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 6c2933a8d72f9328a2931a8166439bed96b80f24 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:04:52 -0700 Subject: [PATCH 003/113] add position --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e52e8dcf2..11f2e31ce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,11 +1370,11 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default values, then all is well. - But if the function arguments don't have default values, then evaluation fails. + But if the function arguments don't have default values, evaluation fails. The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. From 9f2b25ce55e38fc5772fcb65bd98651255f6a49b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:17:47 -0700 Subject: [PATCH 004/113] remove unused ftn; reformat line breaks --- src/libexpr/eval.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 11f2e31ce..0446376f3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -636,11 +636,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(s, s1); -} - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ @@ -1371,12 +1366,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. If that - nix expression is a function, and that function's arguments all have default values, then all is well. + An 'auto-call' is when a nix expression is evaluated without any external arguments. + If that nix expression is a function, and that function's arguments all have default + values, then all is well. But if the function arguments don't have default values, evaluation fails. - The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + The classic case for this error is evaluating a nix file with nix-build that expects + to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. { stdenv }: From d8ef423a189b95f9f6e0a385a0f4474bebf297ef Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 19:16:50 -0700 Subject: [PATCH 005/113] error message formatting --- src/libexpr/eval.cc | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0446376f3..2b697882b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1366,29 +1366,30 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. - If that nix expression is a function, and that function's arguments all have default - values, then all is well. - But if the function arguments don't have default values, evaluation fails. +An 'auto-call' is when a nix expression is evaluated without any external arguments. +If that nix expression is a function, and that function's arguments all have default +values, then all is well. - The classic case for this error is evaluating a nix file with nix-build that expects - to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... +But if the function arguments don't have default values, evaluation fails. - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... +The classic case for this error is evaluating a nix file with nix-build that expects +to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... - See this nix pill for more information re callPackage format: - https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + +More about callPackage: +https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 7d9037035ef8bfe0b7ae00a9e3b139ae83ec8b21 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:21:26 -0700 Subject: [PATCH 006/113] usage example location --- src/libutil/error.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..d42781311 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -38,7 +38,7 @@ namespace nix { ErrorInfo structs are sent to the logger as part of an exception, or directly with the logError or logWarning macros. - See the error-demo.cc program for usage examples. + See libutil/tests/logging.cc for usage examples. */ From 3edfe6090e9e15b205c21b19530607cbdcbbbe7a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:29:32 -0700 Subject: [PATCH 007/113] missing argument error --- src/libexpr/eval.cc | 12 ++++++++++-- src/libexpr/nixexpr.hh | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2b697882b..cf9f6c543 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -660,6 +660,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * }); } +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +{ + throw MissingArgumentError({ + .hint = hintfmt(s, s1), + .errPos = pos + }); +} + LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) { e.addTrace(std::nullopt, s, s2); @@ -1365,7 +1373,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default @@ -1373,7 +1381,7 @@ values, then all is well. But if the function arguments don't have default values, evaluation fails. -The classic case for this error is evaluating a nix file with nix-build that expects +The classic case for this error is evaluating a nix file that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..bf2cd1f15 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); +MakeError(MissingArgumentError, Error); MakeError(RestrictedPathError, Error); From 8895fa70a4b05ddebbb5a23ea96464d5e01345fb Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:05:21 -0700 Subject: [PATCH 008/113] pare down the error message --- src/libexpr/eval.cc | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cf9f6c543..540bfcf7b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1373,31 +1373,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -An 'auto-call' is when a nix expression is evaluated without any external arguments. -If that nix expression is a function, and that function's arguments all have default -values, then all is well. +Nix attempted to evaluate a function as a top level expression; in this case it must have all its +arguments supplied either by default values, or passed explicitly with --arg or --argstr. -But if the function arguments don't have default values, evaluation fails. +https://nixos.org/manual/nix/stable/#ss-functions)", i.name); -The classic case for this error is evaluating a nix file that expects -to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... - - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... - -More about callPackage: -https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From b327de9c2d5182e5814b2e956631b8794b45999b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:09:59 -0700 Subject: [PATCH 009/113] change message --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 540bfcf7b..3667ee6ba 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1375,7 +1375,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -Nix attempted to evaluate a function as a top level expression; in this case it must have all its +nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with --arg or --argstr. https://nixos.org/manual/nix/stable/#ss-functions)", i.name); From 437189e446e16399d347e4430c4d115b4cf2ddf1 Mon Sep 17 00:00:00 2001 From: Lily Ballard Date: Tue, 24 Nov 2020 14:12:32 -0800 Subject: [PATCH 010/113] Escape filename given to nix-shell in shebang mode This prevents spaces or other metacharacters from causing nix-shell to execute the wrong path. Fixes #4229. --- src/nix-build/nix-build.cc | 4 ++-- tests/nix-shell.sh | 18 +++++++++++++++++- tests/shell.nix | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 74fafd426..38048da52 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } } diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 7b2be650a..4775bafb9 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -47,6 +47,14 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode again with metacharacters in the filename. +# First word of filename is chosen to not match any file in the test root. +sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + # Test nix-shell shebang mode for ruby # This uses a fake interpreter that returns the arguments passed # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. @@ -54,7 +62,15 @@ sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shel chmod a+rx $TEST_ROOT/shell.shebang.rb output=$($TEST_ROOT/shell.shebang.rb abc ruby) -[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] + +# Test nix-shell shebang mode for ruby again with metacharacters in the filename. +# Note: fake interpreter only space-separates args without adding escapes to its output. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/shell.nix b/tests/shell.nix index 6ce59b416..24ebcc04c 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -50,7 +50,7 @@ let pkgs = rec { # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" {} '' mkdir -p $out/bin - echo 'printf -- "$*"' > $out/bin/ruby + echo 'printf %s "$*"' > $out/bin/ruby chmod a+rx $out/bin/ruby ''; From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: [PATCH 011/113] Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: [PATCH 012/113] Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: [PATCH 013/113] Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin") From fa307875e961a616a049206645a651a76a050a79 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 11 Dec 2020 23:32:45 +0100 Subject: [PATCH 014/113] Introduce NormalType for the normal type of a Value This will be useful to abstract over the ValueType implementation details Make use of it already to replace the showType(ValueType) function --- src/libexpr/eval-cache.cc | 4 ++-- src/libexpr/eval.cc | 34 ++++++++++++++++------------------ src/libexpr/eval.hh | 2 +- src/libexpr/flake/flake.cc | 26 +++++++++++++------------- src/libexpr/value.hh | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 34 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..a11327f77 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -513,7 +513,7 @@ std::string AttrCursor::getString() auto & v = forceValue(); if (v.type != tString && v.type != tPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); return v.type == tString ? v.string.s : v.path; } @@ -548,7 +548,7 @@ string_t AttrCursor::getStringWithContext() else if (v.type == tPath) return {v.path, {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); } bool AttrCursor::getBool() diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c6f4d1716..48fe0bbda 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -165,25 +165,20 @@ const Value *getPrimOp(const Value &v) { return primOp; } - -string showType(ValueType type) +string showType(NormalType type) { switch (type) { - case tInt: return "an integer"; - case tBool: return "a Boolean"; - case tString: return "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList1: case tList2: case tListN: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; - case tExternal: return "an external value"; - case tFloat: return "a float"; + case nInt: return "an integer"; + case nBool: return "a Boolean"; + case nString: return "a string"; + case nPath: return "a path"; + case nNull: return "null"; + case nAttrs: return "a set"; + case nList: return "a list"; + case nFunction: return "a function"; + case nExternal: return "an external value"; + case nFloat: return "a float"; + case nThunk: return "a thunk"; } abort(); } @@ -198,8 +193,11 @@ string showType(const Value & v) case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tBlackhole: return "a black hole"; default: - return showType(v.type); + return showType(v.normalType()); } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e1f61baa..211529954 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -346,7 +346,7 @@ private: /* Return a string representing the type of the value `v'. */ -string showType(ValueType type); +string showType(NormalType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair parseFlakeInputs( @@ -93,7 +93,7 @@ static std::map parseFlakeInputs( static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const Pos & pos) { - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); FlakeInput input; @@ -108,16 +108,16 @@ static FlakeInput parseFlakeInput(EvalState & state, for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); url = attr.value->string.s; attrs.emplace("url", *url); } else if (attr.name == sFlake) { - expectType(state, tBool, *attr.value, *attr.pos); + expectType(state, nBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); } else if (attr.name == sFollows) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { if (attr.value->type == tString) @@ -158,7 +158,7 @@ static std::map parseFlakeInputs( { std::map inputs; - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); for (nix::Attr & inputAttr : *(*value).attrs) { inputs.emplace(inputAttr.name, @@ -199,10 +199,10 @@ static Flake getFlake( Value vInfo; state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack - expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); if (auto description = vInfo.attrs->get(state.sDescription)) { - expectType(state, tString, *description->value, *description->pos); + expectType(state, nString, *description->value, *description->pos); flake.description = description->value->string.s; } @@ -214,9 +214,9 @@ static Flake getFlake( auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { - expectType(state, tLambda, *outputs->value, *outputs->pos); + expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { @@ -231,7 +231,7 @@ static Flake getFlake( auto sNixConfig = state.symbols.create("nixConfig"); if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { - expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); + expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos); for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index fe11bb2ed..833af0f3d 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -29,6 +29,22 @@ typedef enum { tFloat } ValueType; +// This type abstracts over all actual value types in the language, +// grouping together implementation details like tList*, different function +// types, and types in non-normal form (so thunks and co.) +typedef enum { + nThunk, + nInt, + nFloat, + nBool, + nString, + nPath, + nNull, + nAttrs, + nList, + nFunction, + nExternal +} NormalType; class Bindings; struct Env; @@ -147,6 +163,26 @@ struct Value NixFloat fpoint; }; + // Returns the normal type of a Value. This only returns nThunk if the + // Value hasn't been forceValue'd + inline NormalType normalType() const + { + switch (type) { + case tInt: return nInt; + case tBool: return nBool; + case tString: return nString; + case tPath: return nPath; + case tNull: return nNull; + case tAttrs: return nAttrs; + case tList1: case tList2: case tListN: return nList; + case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; + case tExternal: return nExternal; + case tFloat: return nFloat; + case tThunk: case tApp: case tBlackhole: return nThunk; + } + abort(); + } + bool isList() const { return type == tList1 || type == tList2 || type == tListN; From 9f056f7afdb85b8c3bd59638197e356f269129b2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 00:19:05 +0100 Subject: [PATCH 015/113] Introduce Value type setters and make use of them --- src/libexpr/attr-set.cc | 2 +- src/libexpr/eval-inline.hh | 4 ++-- src/libexpr/eval.cc | 24 ++++++++++++------------ src/libexpr/value.hh | 35 +++++++++++++++++++++++++++-------- src/nix/repl.cc | 2 +- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b1d61a285..17886a426 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -25,7 +25,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) return; } clearValue(v); - v.type = tAttrs; + v.setAttrs(); v.attrs = allocBindings(capacity); nrAttrsets++; nrAttrsInAttrsets += capacity; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 30f6ec7db..a0fd9b569 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -36,11 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos) Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.type = tBlackhole; + v.setBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.type = tThunk; + v.setThunk(); v.thunk.env = env; v.thunk.expr = expr; throw; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 48fe0bbda..f68828944 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -431,7 +431,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) } clearValue(vEmptySet); - vEmptySet.type = tAttrs; + vEmptySet.setAttrs(); vEmptySet.attrs = allocBindings(0); createBaseEnv(); @@ -548,7 +548,7 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; + vPrimOp->setPrimOp(); vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; Value v; mkApp(v, *vPrimOp, *vPrimOp); @@ -556,7 +556,7 @@ Value * EvalState::addPrimOp(const string & name, } Value * v = allocValue(); - v->type = tPrimOp; + v->setPrimOp(); v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; @@ -572,7 +572,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; + vPrimOp->setPrimOp(); vPrimOp->primOp = new PrimOp(std::move(primOp)); Value v; mkApp(v, *vPrimOp, *vPrimOp); @@ -584,7 +584,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) primOp.name = symbols.create(std::string(primOp.name, 2)); Value * v = allocValue(); - v->type = tPrimOp; + v->setPrimOp(); v->primOp = new PrimOp(std::move(primOp)); staticBaseEnv.vars[envName] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; @@ -714,7 +714,7 @@ void mkString(Value & v, const char * s) Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.type = tString; + v.setString(); v.string.s = dupStringWithLen(s.data(), s.size()); v.string.context = 0; if (!context.empty()) { @@ -794,11 +794,11 @@ void EvalState::mkList(Value & v, size_t size) { clearValue(v); if (size == 1) - v.type = tList1; + v.setList1(); else if (size == 2) - v.type = tList2; + v.setList2(); else { - v.type = tListN; + v.setListN(); v.bigList.size = size; v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; } @@ -810,7 +810,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.type = tThunk; + v.setThunk(); v.thunk.env = &env; v.thunk.expr = expr; nrThunks++; @@ -1207,7 +1207,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.type = tLambda; + v.setLambda(); v.lambda.env = &env; v.lambda.fun = this; } @@ -1252,7 +1252,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.type = tPrimOpApp; + v.setPrimOpApp(); v.primOpApp.left = fun2; v.primOpApp.right = &arg; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 833af0f3d..0995dcd7b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -107,6 +107,25 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { ValueType type; + + inline void setInt() { type = tInt; }; + inline void setBool() { type = tBool; }; + inline void setString() { type = tString; }; + inline void setPath() { type = tPath; }; + inline void setNull() { type = tNull; }; + inline void setAttrs() { type = tAttrs; }; + inline void setList1() { type = tList1; }; + inline void setList2() { type = tList2; }; + inline void setListN() { type = tListN; }; + inline void setThunk() { type = tThunk; }; + inline void setApp() { type = tApp; }; + inline void setLambda() { type = tLambda; }; + inline void setBlackhole() { type = tBlackhole; }; + inline void setPrimOp() { type = tPrimOp; }; + inline void setPrimOpApp() { type = tPrimOpApp; }; + inline void setExternal() { type = tExternal; }; + inline void setFloat() { type = tFloat; }; + union { NixInt integer; @@ -223,7 +242,7 @@ static inline void clearValue(Value & v) static inline void mkInt(Value & v, NixInt n) { clearValue(v); - v.type = tInt; + v.setInt(); v.integer = n; } @@ -231,7 +250,7 @@ static inline void mkInt(Value & v, NixInt n) static inline void mkFloat(Value & v, NixFloat n) { clearValue(v); - v.type = tFloat; + v.setFloat(); v.fpoint = n; } @@ -239,7 +258,7 @@ static inline void mkFloat(Value & v, NixFloat n) static inline void mkBool(Value & v, bool b) { clearValue(v); - v.type = tBool; + v.setBool(); v.boolean = b; } @@ -247,13 +266,13 @@ static inline void mkBool(Value & v, bool b) static inline void mkNull(Value & v) { clearValue(v); - v.type = tNull; + v.setNull(); } static inline void mkApp(Value & v, Value & left, Value & right) { - v.type = tApp; + v.setApp(); v.app.left = &left; v.app.right = &right; } @@ -261,7 +280,7 @@ static inline void mkApp(Value & v, Value & left, Value & right) static inline void mkPrimOpApp(Value & v, Value & left, Value & right) { - v.type = tPrimOpApp; + v.setPrimOpApp(); v.app.left = &left; v.app.right = &right; } @@ -269,7 +288,7 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right) static inline void mkStringNoCopy(Value & v, const char * s) { - v.type = tString; + v.setString(); v.string.s = s; v.string.context = 0; } @@ -287,7 +306,7 @@ void mkString(Value & v, const char * s); static inline void mkPathNoCopy(Value & v, const char * s) { clearValue(v); - v.type = tPath; + v.setPath(); v.path = s; } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 71794a309..3cee81b49 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -551,7 +551,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.type = tThunk; + v.setThunk(); v.thunk.env = env; v.thunk.expr = e; addVarToScope(state->symbols.create(name), v); From 22ead43a0b8f94f5a4fb64cff14bf6a2a35d671c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:09:10 +0100 Subject: [PATCH 016/113] Use Value::normalType on all forced values instead of Value::type --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 22 ++--- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 112 +++++++++++++------------- src/libexpr/flake/flake.cc | 12 +-- src/libexpr/get-drvs.cc | 30 +++---- src/libexpr/primops.cc | 89 +++++++++----------- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 12 +-- src/libexpr/value-to-json.cc | 25 +++--- src/libexpr/value-to-xml.cc | 33 ++++---- src/nix-env/nix-env.cc | 16 ++-- src/nix/eval.cc | 4 +- src/nix/flake.cc | 2 +- src/nix/repl.cc | 47 +++++------ 15 files changed, 199 insertions(+), 213 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 83854df49..54e13e6a2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -67,7 +67,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (apType == apAttr) { - if (v->type != tAttrs) + if (v->normalType() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a11327f77..3c97f1201 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -390,14 +390,14 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.type == tString) + if (v.normalType() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == tPath) + else if (v.normalType() == nPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; - else if (v.type == tBool) + else if (v.normalType() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.type == tAttrs) + else if (v.normalType() == nAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; @@ -442,7 +442,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro auto & v = forceValue(); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); @@ -512,10 +512,10 @@ std::string AttrCursor::getString() auto & v = forceValue(); - if (v.type != tString && v.type != tPath) + if (v.normalType() != nString && v.normalType() != nPath) throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); - return v.type == tString ? v.string.s : v.path; + return v.normalType() == nString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() @@ -543,9 +543,9 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type == tString) + if (v.normalType() == nString) return {v.string.s, v.getContext()}; - else if (v.type == tPath) + else if (v.normalType() == nPath) return {v.path, {}}; else throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); @@ -567,7 +567,7 @@ bool AttrCursor::getBool() auto & v = forceValue(); - if (v.type != tBool) + if (v.normalType() != nBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; @@ -589,7 +589,7 @@ std::vector AttrCursor::getAttrs() auto & v = forceValue(); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector attrs; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index a0fd9b569..9b644d5cb 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f68828944..f33426b59 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.type != tAttrs) + if (v.normalType() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.type == tNull) + if (nameVal.normalType() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->type != tAttrs || + if (vAttrs->normalType() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->type != tAttrs || + if (vAttrs->normalType() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po return; } - if (fun.type == tAttrs) { + if (fun.normalType() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.type == tAttrs) { + if (fun.normalType() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - ValueType firstType = tString; + NormalType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1573,36 +1573,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.type; + firstType = vTmp.normalType(); first = false; } - if (firstType == tInt) { - if (vTmp.type == tInt) { + if (firstType == nInt) { + if (vTmp.normalType() == nInt) { n += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.normalType() == nFloat) { // Upgrade the type from int to float; - firstType = tFloat; + firstType = nFloat; nf = n; nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); - } else if (firstType == tFloat) { - if (vTmp.type == tInt) { + } else if (firstType == nFloat) { + if (vTmp.normalType() == nInt) { nf += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.normalType() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else - s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + s << state.coerceToString(pos, vTmp, context, false, firstType == nString); } - if (firstType == tInt) + if (firstType == nInt) mkInt(v, n); - else if (firstType == tFloat) + else if (firstType == nFloat) mkFloat(v, nf); - else if (firstType == tPath) { + else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); @@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.type == tAttrs) { + if (v.normalType() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tInt) + if (v.normalType() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type == tInt) + if (v.normalType() == nInt) return v.integer; - else if (v.type != tFloat) + else if (v.normalType() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tBool) + if (v.normalType() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) + if (v.normalType() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tString) { + if (v.normalType() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.type != tAttrs) return false; + if (v.normalType() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->type != tString) return false; + if (i->value->normalType() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string s; - if (v.type == tString) { + if (v.normalType() == nString) { copyContext(v, context); return v.string.s; } - if (v.type == tPath) { + if (v.normalType() == nPath) { Path path(canonPath(v.path)); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.type == tAttrs) { + if (v.normalType() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.type == tExternal) + if (v.normalType() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return std::to_string(v.integer); - if (v.type == tFloat) return std::to_string(v.fpoint); - if (v.type == tNull) return ""; + if (v.normalType() == nBool && v.boolean) return "1"; + if (v.normalType() == nBool && !v.boolean) return ""; + if (v.normalType() == nInt) return std::to_string(v.integer); + if (v.normalType() == nFloat) return std::to_string(v.fpoint); + if (v.normalType() == nNull) return ""; if (v.isList()) { string result; @@ -1884,40 +1884,38 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.type == tInt && v2.type == tFloat) + if (v1.normalType() == nInt && v2.normalType() == nFloat) return v1.integer == v2.fpoint; - if (v1.type == tFloat && v2.type == tInt) + if (v1.normalType() == nFloat && v2.normalType() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.type != v2.type) return false; + if (v1.normalType() != v2.normalType()) return false; - switch (v1.type) { + switch (v1.normalType()) { - case tInt: + case nInt: return v1.integer == v2.integer; - case tBool: + case nBool: return v1.boolean == v2.boolean; - case tString: + case nString: return strcmp(v1.string.s, v2.string.s) == 0; - case tPath: + case nPath: return strcmp(v1.path, v2.path) == 0; - case tNull: + case nNull: return true; - case tList1: - case tList2: - case tListN: + case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; - case tAttrs: { + case nAttrs: { /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { @@ -1939,15 +1937,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) } /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: + case nFunction: return false; - case tExternal: + case nExternal: return *v1.external == *v2.external; - case tFloat: + case nFloat: return v1.fpoint == v2.fpoint; default: diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 881b1b4e5..c126b2c40 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type == tString) + if (attr.value->normalType() == nString) attrs.emplace(attr.name, attr.value->string.s); else throw TypeError("flake input attribute '%s' is %s while a string is expected", @@ -235,17 +235,17 @@ static Flake getFlake( for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); - if (setting.value->type == tString) + if (setting.value->normalType() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); - else if (setting.value->type == tInt) + else if (setting.value->normalType() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); - else if (setting.value->type == tBool) + else if (setting.value->normalType() == nBool) flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); - else if (setting.value->isList()) { + else if (setting.value->normalType() == nList) { std::vector ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { auto elem = setting.value->listElems()[n]; - if (elem->type != tString) + if (elem->normalType() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 91916e8bf..93788273f 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type != tString) throw errMsg; + if ((*i)->normalType() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.isList()) { + if (v.normalType() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.type == tInt || v.type == tBool || v.type == tString || - v.type == tFloat; + else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString || + v.normalType() == nFloat; } @@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; + if (!v || v->normalType() != nString) return ""; return v->string.s; } @@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { + if (v->normalType() == nInt) return v->integer; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ NixInt n; @@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tFloat) return v->fpoint; - if (v->type == tString) { + if (v->normalType() == nFloat) return v->fpoint; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ NixFloat n; @@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { + if (v->normalType() == nBool) return v->boolean; + if (v->normalType() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->type == tAttrs) { + if (i->value->normalType() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.isList()) { + else if (v.normalType() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 41f06c219..d501f7482 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -356,24 +356,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList1: case tList2: case tListN: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - case tExternal: + switch (args[0]->normalType()) { + case nInt: t = "int"; break; + case nBool: t = "bool"; break; + case nString: t = "string"; break; + case nPath: t = "path"; break; + case nNull: t = "null"; break; + case nAttrs: t = "set"; break; + case nList: t = "list"; break; + case nFunction: t = "lambda"; break; + case nExternal: t = args[0]->external->typeOf(); break; - case tFloat: t = "float"; break; - default: abort(); + case nFloat: t = "float"; break; + case nThunk: abort(); } mkString(v, state.symbols.create(t)); } @@ -393,7 +389,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tNull); + mkBool(v, args[0]->normalType() == nNull); } static RegisterPrimOp primop_isNull({ @@ -413,18 +409,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - bool res; - switch (args[0]->type) { - case tLambda: - case tPrimOp: - case tPrimOpApp: - res = true; - break; - default: - res = false; - break; - } - mkBool(v, res); + mkBool(v, args[0]->normalType() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -440,7 +425,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tInt); + mkBool(v, args[0]->normalType() == nInt); } static RegisterPrimOp primop_isInt({ @@ -456,7 +441,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tFloat); + mkBool(v, args[0]->normalType() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -472,7 +457,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tString); + mkBool(v, args[0]->normalType() == nString); } static RegisterPrimOp primop_isString({ @@ -488,7 +473,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tBool); + mkBool(v, args[0]->normalType() == nBool); } static RegisterPrimOp primop_isBool({ @@ -504,7 +489,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tPath); + mkBool(v, args[0]->normalType() == nPath); } static RegisterPrimOp primop_isPath({ @@ -520,20 +505,20 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->type == tFloat && v2->type == tInt) + if (v1->normalType() == nFloat && v2->normalType() == nInt) return v1->fpoint < v2->integer; - if (v1->type == tInt && v2->type == tFloat) + if (v1->normalType() == nInt && v2->normalType() == nFloat) return v1->integer < v2->fpoint; - if (v1->type != v2->type) + if (v1->normalType() != v2->normalType()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->type) { - case tInt: + switch (v1->normalType()) { + case nInt: return v1->integer < v2->integer; - case tFloat: + case nFloat: return v1->fpoint < v2->fpoint; - case tString: + case nString: return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: + case nPath: return strcmp(v1->path, v2->path) < 0; default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); @@ -777,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tString) + if (args[0]->normalType() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -902,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->type == tNull) continue; + if (i->value->normalType() == nNull) continue; } if (i->name == state.sContentAddressed) { @@ -1308,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1808,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args }); state.forceValue(*args[0], pos); - if (args[0]->type != tLambda) + if (args[0]->normalType() != nFunction) throw TypeError({ .hint = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", @@ -2074,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tAttrs); + mkBool(v, args[0]->normalType() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2337,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->isList()); + mkBool(v, args[0]->normalType() == nList); } static RegisterPrimOp primop_isList({ @@ -2831,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); @@ -2850,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); @@ -2869,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); @@ -2896,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - if (args[0]->type == tFloat || args[1]->type == tFloat) { + if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a77035c16..2461ebc99 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..6d93e1dc2 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,25 +85,25 @@ static void fetchTree( state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tPath || attr.value->type == tString) + if (attr.value->normalType() == nPath || attr.value->normalType() == nString) addURI( state, attrs, attr.name, state.coerceToString(*attr.pos, *attr.value, context, false, false) ); - else if (attr.value->type == tString) + else if (attr.value->normalType() == nString) addURI(state, attrs, attr.name, attr.value->string.s); - else if (attr.value->type == tBool) + else if (attr.value->normalType() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->type == tInt) + else if (attr.value->normalType() == nInt) attrs.emplace(attr.name, attr.value->integer); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", @@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->normalType() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 6ec8315ba..b5f4c8654 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: out.write(v.integer); break; - case tBool: + case nBool: out.write(v.boolean); break; - case tString: + case nString: copyContext(v, context); out.write(v.string.s); break; - case tPath: + case nPath: out.write(state.copyPathToStore(context, v.path)); break; - case tNull: + case nNull: out.write(nullptr); break; - case tAttrs: { + case nAttrs: { auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); if (maybeString) { out.write(*maybeString); @@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tList1: case tList2: case tListN: { + case nList: { auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); @@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tExternal: + case nExternal: v.external->printValueAsJSON(state, strict, out, context); break; - case tFloat: + case nFloat: out.write(v.fpoint); break; - default: + case nThunk: + throw TypeError("cannot convert %1% to JSON", showType(v)); + + case nFunction: throw TypeError("cannot convert %1% to JSON", showType(v)); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1f0b1541d..26be07cff 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); break; - case tBool: + case nBool: doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); break; - case tString: + case nString: /* !!! show the context? */ copyContext(v, context); doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); break; - case tPath: + case nPath: doc.writeEmptyElement("path", singletonAttrs("value", v.path)); break; - case tNull: + case nNull: doc.writeEmptyElement("null"); break; - case tAttrs: + case nAttrs: if (state.isDerivation(v)) { XMLAttrs xmlAttrs; @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->normalType() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->normalType() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; - case tList1: case tList2: case tListN: { + case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); break; } - case tLambda: { + case nFunction: { + if (!v.isLambda()) { + // FIXME: Serialize primops and primopapps + doc.writeEmptyElement("unevaluated"); + break; + } XMLAttrs xmlAttrs; if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); @@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } - case tExternal: + case nExternal: v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); break; - case tFloat: + case nFloat: doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); break; - default: + case nThunk: doc.writeEmptyElement("unevaluated"); } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a4b5c9e2c..404fd5111 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) i.queryName(), j) }); else { - if (v->type == tString) { + if (v->normalType() == nString) { attrs2["type"] = "string"; attrs2["value"] = v->string.s; xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tInt) { + } else if (v->normalType() == nInt) { attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tFloat) { + } else if (v->normalType() == nFloat) { attrs2["type"] = "float"; attrs2["value"] = (format("%1%") % v->fpoint).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tBool) { + } else if (v->normalType() == nBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->isList()) { + } else if (v->normalType() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type != tString) continue; + if (v->listElems()[j]->normalType() != nString) continue; XMLAttrs attrs3; attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } - } else if (v->type == tAttrs) { + } else if (v->normalType() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); Bindings & attrs = *v->attrs; for (auto &i : attrs) { Attr & a(*attrs.find(i.name)); - if(a.value->type != tString) continue; + if(a.value->normalType() != nString) continue; XMLAttrs attrs3; attrs3["type"] = i.name; attrs3["value"] = a.value->string.s; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 0f02919de..bba3b1bc6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand recurse = [&](Value & v, const Pos & pos, const Path & path) { state->forceValue(v); - if (v.type == tString) + if (v.normalType() == nString) // FIXME: disallow strings with contexts? writeFile(path, v.string.s); - else if (v.type == tAttrs) { + else if (v.normalType() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7a7c71676..80b050091 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand if (v.type == tLambda) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type == tAttrs) { + } else if (v.normalType() == nAttrs) { for (auto & attr : *v.attrs) try { state->forceValue(*attr.value, *attr.pos); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 3cee81b49..56184efb9 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -446,7 +446,7 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.type == tPath || v.type == tString) { + if (v.normalType() == nPath || v.normalType() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); @@ -669,31 +669,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.type) { + switch (v.normalType()) { - case tInt: + case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; break; - case tBool: + case nBool: str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; break; - case tString: + case nString: str << ANSI_YELLOW; printStringValue(str, v.string.s); str << ANSI_NORMAL; break; - case tPath: + case nPath: str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? break; - case tNull: + case nNull: str << ANSI_CYAN "null" ANSI_NORMAL; break; - case tAttrs: { + case nAttrs: { seen.insert(&v); bool isDrv = state->isDerivation(v); @@ -738,9 +738,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; } - case tList1: - case tList2: - case tListN: + case nList: seen.insert(&v); str << "[ "; @@ -761,22 +759,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "]"; break; - case tLambda: { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - break; - } - - case tPrimOp: - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + case nFunction: + if (v.type == tLambda) { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; + } else if (v.type == tPrimOp) { + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + } else if (v.type == tPrimOpApp) { + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + } else { + abort(); + } break; - case tPrimOpApp: - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; - break; - - case tFloat: + case nFloat: str << v.fpoint; break; From bf9890396731a2bbe4f04a49684dee463d818906 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:15:11 +0100 Subject: [PATCH 017/113] Add ValueType checking functions for types that have the same NormalType --- src/libexpr/eval-inline.hh | 6 +++--- src/libexpr/eval.cc | 20 ++++++++++---------- src/libexpr/flake/flake.cc | 4 ++-- src/libexpr/primops.cc | 6 +++--- src/libexpr/value.hh | 14 ++++++++++++++ src/nix/flake.cc | 6 +++--- src/nix/main.cc | 2 +- src/nix/repl.cc | 8 ++++---- 8 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 9b644d5cb..8c40c2565 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -32,7 +32,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const void EvalState::forceValue(Value & v, const Pos & pos) { - if (v.type == tThunk) { + if (v.isThunk()) { Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { @@ -46,9 +46,9 @@ void EvalState::forceValue(Value & v, const Pos & pos) throw; } } - else if (v.type == tApp) + else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.type == tBlackhole) + else if (v.isBlackhole()) throwEvalError(pos, "infinite recursion encountered"); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f33426b59..5f9d19b8d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -158,10 +158,10 @@ std::ostream & operator << (std::ostream & str, const Value & v) const Value *getPrimOp(const Value &v) { const Value * primOp = &v; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); return primOp; } @@ -601,9 +601,9 @@ Value & EvalState::getBuiltin(const string & name) std::optional EvalState::getDoc(Value & v) { - if (v.type == tPrimOp || v.type == tPrimOpApp) { + if (v.isPrimOp() || v.isPrimOpApp()) { auto v2 = &v; - while (v2->type == tPrimOpApp) + while (v2->isPrimOpApp()) v2 = v2->primOpApp.left; if (v2->primOp->doc) return Doc { @@ -1227,11 +1227,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* Figure out the number of arguments still needed. */ size_t argsDone = 0; Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { argsDone++; primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); auto arity = primOp->primOp->arity; auto argsLeft = arity - argsDone; @@ -1242,7 +1242,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) Value * vArgs[arity]; auto n = arity - 1; vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; /* And call the primop. */ @@ -1264,7 +1264,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po forceValue(fun, pos); - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + if (fun.isPrimOp() || fun.isPrimOpApp()) { callPrimOp(fun, arg, v, pos); return; } @@ -1285,7 +1285,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } } - if (fun.type != tLambda) + if (!fun.isLambda()) throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1378,7 +1378,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { res = fun; return; } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c126b2c40..2f9658ab8 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -73,7 +73,7 @@ static std::tuple fetchOrSubstituteTree( static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) { - if (value.type == tThunk && value.isTrivial()) + if (value.isThunk() && value.isTrivial()) state.forceValue(value, pos); } @@ -216,7 +216,7 @@ static Flake getFlake( if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d501f7482..f6ca612f4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2239,11 +2239,11 @@ static RegisterPrimOp primop_catAttrs({ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { state.mkAttrs(v, 0); return; } - if (args[0]->type != tLambda) + if (!args[0]->isLambda()) throw TypeError({ .hint = hintfmt("'functionArgs' requires a function"), .errPos = pos @@ -2674,7 +2674,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) return CompareValues()(a, b); Value vTmp1, vTmp2; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 0995dcd7b..e743da9c3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -126,6 +126,20 @@ struct Value inline void setExternal() { type = tExternal; }; inline void setFloat() { type = tFloat; }; + // Functions needed to distinguish the type + // These should be removed eventually, by putting the functionality that's + // needed by callers into methods of this type + + // normalType() == nThunk + inline bool isThunk() const { return type == tThunk; }; + inline bool isApp() const { return type == tApp; }; + inline bool isBlackhole() const { return type == tBlackhole; }; + + // normalType() == nFunction + inline bool isLambda() const { return type == tLambda; }; + inline bool isPrimOp() const { return type == tPrimOp; }; + inline bool isPrimOpApp() const { return type == tPrimOpApp; }; + union { NixInt integer; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 80b050091..e4da0348c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -260,7 +260,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); if (!body || body->matchAttrs || std::string(body->arg) != "prev") @@ -276,7 +276,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type == tLambda) { + if (v.isLambda()) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); } else if (v.normalType() == nAttrs) { @@ -371,7 +371,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda) + if (!v.isLambda()) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || diff --git a/src/nix/main.cc b/src/nix/main.cc index 27b1d7257..e7a15dec9 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -272,7 +272,7 @@ void mainWrapped(int argc, char * * argv) auto builtins = state.baseEnv.values[0]->attrs; for (auto & builtin : *builtins) { auto b = nlohmann::json::object(); - if (builtin.value->type != tPrimOp) continue; + if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp; if (!primOp->doc) continue; b["arity"] = primOp->arity; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 56184efb9..047e2dc59 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -450,7 +450,7 @@ bool NixRepl::processLine(string line) PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); - } else if (v.type == tLambda) { + } else if (v.isLambda()) { pos = v.lambda.fun->pos; } else { // assume it's a derivation @@ -760,13 +760,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nFunction: - if (v.type == tLambda) { + if (v.isLambda()) { std::ostringstream s; s << v.lambda.fun->pos; str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - } else if (v.type == tPrimOp) { + } else if (v.isPrimOp()) { str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - } else if (v.type == tPrimOpApp) { + } else if (v.isPrimOpApp()) { str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; } else { abort(); From 730b152b190135adef2f53c7a80cfd1111d37ead Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 12 Dec 2020 02:22:58 +0100 Subject: [PATCH 018/113] Make Value::type private This is an implementation detail and shouldn't be used. Use normalType() and the various is functions instead --- src/libexpr/eval.cc | 2 +- src/libexpr/value.hh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5f9d19b8d..1d11039ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -68,7 +68,7 @@ RootValue allocRootValue(Value * v) } -static void printValue(std::ostream & str, std::set & active, const Value & v) +void printValue(std::ostream & str, std::set & active, const Value & v) { checkInterrupt(); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index e743da9c3..4050d7e4b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -106,8 +106,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { +private: ValueType type; +friend std::string showType(const Value & v); +friend void printValue(std::ostream & str, std::set & active, const Value & v); + +public: + inline void setInt() { type = tInt; }; inline void setBool() { type = tBool; }; inline void setString() { type = tString; }; From 7080321618e29033a8b5dc2f9fc938dcf2df270d Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 10:54:24 +0100 Subject: [PATCH 019/113] Use the fs accessor for readInvalidDerivation Extend `FSAccessor::readFile` to allow not checking that the path is a valid one, and rewrite `readInvalidDerivation` using this extended `readFile`. Several places in the code use `readInvalidDerivation`, either because they need to read a derivation that has been written in the store but not registered yet, or more generally to prevent a deadlock because `readDerivation` tries to lock the state, so can't be called from a place where the lock is already held. However, `readInvalidDerivation` implicitely assumes that the store is a `LocalFSStore`, which isn't always the case. The concrete motivation for this is that it's required for `nix copy --from someBinaryCache` to work, which is tremendously useful for the tests. --- src/libstore/fs-accessor.hh | 9 ++++++++- src/libstore/local-fs-store.cc | 8 ++++---- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.hh | 4 ++-- src/libstore/store-api.cc | 21 +++++++++------------ 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 64780a6da..c825e84f2 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -25,7 +25,14 @@ public: virtual StringSet readDirectory(const Path & path) = 0; - virtual std::string readFile(const Path & path) = 0; + /** + * Read a file inside the store. + * + * If `requireValidPath` is set to `true` (the default), the path must be + * inside a valid store path, otherwise it just needs to be physically + * present (but not necessarily properly registered) + */ + virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; virtual std::string readLink(const Path & path) = 0; }; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index e7c3dae92..6de13c73a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor LocalStoreAccessor(ref store) : store(store) { } - Path toRealPath(const Path & path) + Path toRealPath(const Path & path, bool requireValidPath = true) { auto storePath = store->toStorePath(path).first; - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { - return nix::readFile(toRealPath(path)); + return nix::readFile(toRealPath(path, requireValidPath)); } std::string readLink(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1427a0f98..784ebb719 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 63bde92de..f43456f0b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & } } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) +std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) { auto path = canonPath(path_); auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); auto i = nars.find(std::string(storePath.hashPart())); @@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path) return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path) +std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) { - auto res = fetch(path); + auto res = fetch(path, requireValidPath); return res.first->readFile(res.second); } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 347cf5764..594852d0e 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor Path cacheDir; - std::pair, Path> fetch(const Path & path_); + std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); friend class BinaryCacheStore; @@ -32,7 +32,7 @@ public: StringSet readDirectory(const Path & path) override; - std::string readFile(const Path & path) override; + std::string readFile(const Path & path, bool requireValidPath = true) override; std::string readLink(const Path & path) override; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..25e28cffa 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1018,26 +1018,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } - -Derivation Store::readDerivation(const StorePath & drvPath) +Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) { - auto accessor = getFSAccessor(); + auto accessor = store.getFSAccessor(); try { - return parseDerivation(*this, - accessor->readFile(printStorePath(drvPath)), + return parseDerivation(store, + accessor->readFile(store.printStorePath(drvPath), requireValidPath), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } } +Derivation Store::readDerivation(const StorePath & drvPath) +{ return readDerivationCommon(*this, drvPath, true); } + Derivation Store::readInvalidDerivation(const StorePath & drvPath) -{ - return parseDerivation( - *this, - readFile(Store::toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)); -} +{ return readDerivationCommon(*this, drvPath, false); } } From d67e02919c7f941615407dfd14cfdab6a28c4c26 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 17 Dec 2020 14:42:52 +0100 Subject: [PATCH 020/113] Rename ValueType -> InternalType, NormalType -> ValueType And Value::type to Value::internalType, such that type() can be used in the next commit to get the new ValueType --- src/libexpr/eval.cc | 16 +++++----- src/libexpr/eval.hh | 2 +- src/libexpr/flake/flake.cc | 2 +- src/libexpr/value.hh | 64 +++++++++++++++++++------------------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d11039ad..e14eb01c7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -77,7 +77,7 @@ void printValue(std::ostream & str, std::set & active, const Valu return; } - switch (v.type) { + switch (v.internalType) { case tInt: str << v.integer; break; @@ -165,7 +165,7 @@ const Value *getPrimOp(const Value &v) { return primOp; } -string showType(NormalType type) +string showType(ValueType type) { switch (type) { case nInt: return "an integer"; @@ -186,7 +186,7 @@ string showType(NormalType type) string showType(const Value & v) { - switch (v.type) { + switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); @@ -205,9 +205,9 @@ string showType(const Value & v) bool Value::isTrivial() const { return - type != tApp - && type != tPrimOpApp - && (type != tThunk + internalType != tApp + && internalType != tPrimOpApp + && (internalType != tThunk || (dynamic_cast(thunk.expr) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) || dynamic_cast(thunk.expr) @@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - NormalType firstType = nString; + ValueType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1728,7 +1728,7 @@ void copyContext(const Value & v, PathSet & context) std::vector> Value::getContext() { std::vector> res; - assert(type == tString); + assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) res.push_back(decodeContext(*p)); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 211529954..0e1f61baa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -346,7 +346,7 @@ private: /* Return a string representing the type of the value `v'. */ -string showType(NormalType type); +string showType(ValueType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair & active, const Value & v); public: - inline void setInt() { type = tInt; }; - inline void setBool() { type = tBool; }; - inline void setString() { type = tString; }; - inline void setPath() { type = tPath; }; - inline void setNull() { type = tNull; }; - inline void setAttrs() { type = tAttrs; }; - inline void setList1() { type = tList1; }; - inline void setList2() { type = tList2; }; - inline void setListN() { type = tListN; }; - inline void setThunk() { type = tThunk; }; - inline void setApp() { type = tApp; }; - inline void setLambda() { type = tLambda; }; - inline void setBlackhole() { type = tBlackhole; }; - inline void setPrimOp() { type = tPrimOp; }; - inline void setPrimOpApp() { type = tPrimOpApp; }; - inline void setExternal() { type = tExternal; }; - inline void setFloat() { type = tFloat; }; + inline void setInt() { internalType = tInt; }; + inline void setBool() { internalType = tBool; }; + inline void setString() { internalType = tString; }; + inline void setPath() { internalType = tPath; }; + inline void setNull() { internalType = tNull; }; + inline void setAttrs() { internalType = tAttrs; }; + inline void setList1() { internalType = tList1; }; + inline void setList2() { internalType = tList2; }; + inline void setListN() { internalType = tListN; }; + inline void setThunk() { internalType = tThunk; }; + inline void setApp() { internalType = tApp; }; + inline void setLambda() { internalType = tLambda; }; + inline void setBlackhole() { internalType = tBlackhole; }; + inline void setPrimOp() { internalType = tPrimOp; }; + inline void setPrimOpApp() { internalType = tPrimOpApp; }; + inline void setExternal() { internalType = tExternal; }; + inline void setFloat() { internalType = tFloat; }; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type // normalType() == nThunk - inline bool isThunk() const { return type == tThunk; }; - inline bool isApp() const { return type == tApp; }; - inline bool isBlackhole() const { return type == tBlackhole; }; + inline bool isThunk() const { return internalType == tThunk; }; + inline bool isApp() const { return internalType == tApp; }; + inline bool isBlackhole() const { return internalType == tBlackhole; }; // normalType() == nFunction - inline bool isLambda() const { return type == tLambda; }; - inline bool isPrimOp() const { return type == tPrimOp; }; - inline bool isPrimOpApp() const { return type == tPrimOpApp; }; + inline bool isLambda() const { return internalType == tLambda; }; + inline bool isPrimOp() const { return internalType == tPrimOp; }; + inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; union { @@ -204,9 +204,9 @@ public: // Returns the normal type of a Value. This only returns nThunk if the // Value hasn't been forceValue'd - inline NormalType normalType() const + inline ValueType normalType() const { - switch (type) { + switch (internalType) { case tInt: return nInt; case tBool: return nBool; case tString: return nString; @@ -224,22 +224,22 @@ public: bool isList() const { - return type == tList1 || type == tList2 || type == tListN; + return internalType == tList1 || internalType == tList2 || internalType == tListN; } Value * * listElems() { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } const Value * const * listElems() const { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } size_t listSize() const { - return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } /* Check whether forcing this value requires a trivial amount of From 12e65078ef5c511196c9e48f7fdf71f6c0e5c89f Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 17 Dec 2020 14:45:45 +0100 Subject: [PATCH 021/113] Rename Value::normalType() -> Value::type() --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval-cache.cc | 26 +++++----- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 74 +++++++++++++-------------- src/libexpr/flake/flake.cc | 16 +++--- src/libexpr/get-drvs.cc | 30 +++++------ src/libexpr/primops.cc | 44 ++++++++-------- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 12 ++--- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 +-- src/libexpr/value.hh | 6 +-- src/nix-env/nix-env.cc | 16 +++--- src/nix/eval.cc | 4 +- src/nix/flake.cc | 2 +- src/nix/repl.cc | 4 +- 16 files changed, 125 insertions(+), 125 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 54e13e6a2..2d37dcb7e 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -67,7 +67,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (apType == apAttr) { - if (v->normalType() != nAttrs) + if (v->type() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 3c97f1201..75e9af787 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -390,14 +390,14 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.normalType() == nString) + if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.normalType() == nPath) + else if (v.type() == nPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; - else if (v.normalType() == nBool) + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.normalType() == nAttrs) + else if (v.type() == nAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; @@ -442,7 +442,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro auto & v = forceValue(); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); @@ -512,10 +512,10 @@ std::string AttrCursor::getString() auto & v = forceValue(); - if (v.normalType() != nString && v.normalType() != nPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); + if (v.type() != nString && v.type() != nPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); - return v.normalType() == nString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() @@ -543,12 +543,12 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.normalType() == nString) + if (v.type() == nString) return {v.string.s, v.getContext()}; - else if (v.normalType() == nPath) + else if (v.type() == nPath) return {v.path, {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType())); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); } bool AttrCursor::getBool() @@ -567,7 +567,7 @@ bool AttrCursor::getBool() auto & v = forceValue(); - if (v.normalType() != nBool) + if (v.type() != nBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; @@ -589,7 +589,7 @@ std::vector AttrCursor::getAttrs() auto & v = forceValue(); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector attrs; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 8c40c2565..e56ce261c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e14eb01c7..2f8d6d259 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -197,7 +197,7 @@ string showType(const Value & v) case tApp: return "a function application"; case tBlackhole: return "a black hole"; default: - return showType(v.normalType()); + return showType(v.type()); } } @@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.normalType() != nAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.normalType() == nNull) + if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->normalType() != nAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->normalType() != nAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po return; } - if (fun.normalType() == nAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.normalType() == nAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1573,14 +1573,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.normalType(); + firstType = vTmp.type(); first = false; } if (firstType == nInt) { - if (vTmp.normalType() == nInt) { + if (vTmp.type() == nInt) { n += vTmp.integer; - } else if (vTmp.normalType() == nFloat) { + } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; nf = n; @@ -1588,9 +1588,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == nFloat) { - if (vTmp.normalType() == nInt) { + if (vTmp.type() == nInt) { nf += vTmp.integer; - } else if (vTmp.normalType() == nFloat) { + } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); @@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.normalType() == nAttrs) { + if (v.type() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nInt) + if (v.type() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() == nInt) + if (v.type() == nInt) return v.integer; - else if (v.normalType() != nFloat) + else if (v.type() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nFunction && !isFunctor(v)) + if (v.type() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.normalType() != nString) { + if (v.type() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.normalType() != nAttrs) return false; + if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->normalType() != nString) return false; + if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string s; - if (v.normalType() == nString) { + if (v.type() == nString) { copyContext(v, context); return v.string.s; } - if (v.normalType() == nPath) { + if (v.type() == nPath) { Path path(canonPath(v.path)); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.normalType() == nAttrs) { + if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.normalType() == nExternal) + if (v.type() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.normalType() == nBool && v.boolean) return "1"; - if (v.normalType() == nBool && !v.boolean) return ""; - if (v.normalType() == nInt) return std::to_string(v.integer); - if (v.normalType() == nFloat) return std::to_string(v.fpoint); - if (v.normalType() == nNull) return ""; + if (v.type() == nBool && v.boolean) return "1"; + if (v.type() == nBool && !v.boolean) return ""; + if (v.type() == nInt) return std::to_string(v.integer); + if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nNull) return ""; if (v.isList()) { string result; @@ -1884,15 +1884,15 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.normalType() == nInt && v2.normalType() == nFloat) + if (v1.type() == nInt && v2.type() == nFloat) return v1.integer == v2.fpoint; - if (v1.normalType() == nFloat && v2.normalType() == nInt) + if (v1.type() == nFloat && v2.type() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.normalType() != v2.normalType()) return false; + if (v1.type() != v2.type()) return false; - switch (v1.normalType()) { + switch (v1.type()) { case nInt: return v1.integer == v2.integer; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 987e7e24b..4f021570c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -82,9 +82,9 @@ static void expectType(EvalState & state, ValueType type, Value & value, const Pos & pos) { forceTrivialValue(state, value, pos); - if (value.normalType() != type) + if (value.type() != type) throw Error("expected %s but got %s at %s", - showType(type), showType(value.normalType()), pos); + showType(type), showType(value.type()), pos); } static std::map parseFlakeInputs( @@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->normalType() == nString) + if (attr.value->type() == nString) attrs.emplace(attr.name, attr.value->string.s); else throw TypeError("flake input attribute '%s' is %s while a string is expected", @@ -235,17 +235,17 @@ static Flake getFlake( for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); - if (setting.value->normalType() == nString) + if (setting.value->type() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nInt) + else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nBool) + else if (setting.value->type() == nBool) flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); - else if (setting.value->normalType() == nList) { + else if (setting.value->type() == nList) { std::vector ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { auto elem = setting.value->listElems()[n]; - if (elem->normalType() != nString) + if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 93788273f..32c115c12 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->normalType() != nString) throw errMsg; + if ((*i)->type() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.normalType() == nList) { + if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString || - v.normalType() == nFloat; + else return v.type() == nInt || v.type() == nBool || v.type() == nString || + v.type() == nFloat; } @@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->normalType() != nString) return ""; + if (!v || v->type() != nString) return ""; return v->string.s; } @@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nInt) return v->integer; - if (v->normalType() == nString) { + if (v->type() == nInt) return v->integer; + if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ NixInt n; @@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nFloat) return v->fpoint; - if (v->normalType() == nString) { + if (v->type() == nFloat) return v->fpoint; + if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ NixFloat n; @@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->normalType() == nBool) return v->boolean; - if (v->normalType() == nString) { + if (v->type() == nBool) return v->boolean; + if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->normalType() == nAttrs) { + if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.normalType() == nList) { + else if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f6ca612f4..4106f1ec8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -356,7 +356,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->normalType()) { + switch (args[0]->type()) { case nInt: t = "int"; break; case nBool: t = "bool"; break; case nString: t = "string"; break; @@ -389,7 +389,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nNull); + mkBool(v, args[0]->type() == nNull); } static RegisterPrimOp primop_isNull({ @@ -409,7 +409,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nFunction); + mkBool(v, args[0]->type() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -425,7 +425,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nInt); + mkBool(v, args[0]->type() == nInt); } static RegisterPrimOp primop_isInt({ @@ -441,7 +441,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nFloat); + mkBool(v, args[0]->type() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -457,7 +457,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nString); + mkBool(v, args[0]->type() == nString); } static RegisterPrimOp primop_isString({ @@ -473,7 +473,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nBool); + mkBool(v, args[0]->type() == nBool); } static RegisterPrimOp primop_isBool({ @@ -489,7 +489,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nPath); + mkBool(v, args[0]->type() == nPath); } static RegisterPrimOp primop_isPath({ @@ -505,13 +505,13 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->normalType() == nFloat && v2->normalType() == nInt) + if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; - if (v1->normalType() == nInt && v2->normalType() == nFloat) + if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; - if (v1->normalType() != v2->normalType()) + if (v1->type() != v2->type()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->normalType()) { + switch (v1->type()) { case nInt: return v1->integer < v2->integer; case nFloat: @@ -762,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->normalType() == nString) + if (args[0]->type() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -887,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->normalType() == nNull) continue; + if (i->value->type() == nNull) continue; } if (i->name == state.sContentAddressed) { @@ -1293,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1793,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args }); state.forceValue(*args[0], pos); - if (args[0]->normalType() != nFunction) + if (args[0]->type() != nFunction) throw TypeError({ .hint = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", @@ -2059,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nAttrs); + mkBool(v, args[0]->type() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2322,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->normalType() == nList); + mkBool(v, args[0]->type() == nList); } static RegisterPrimOp primop_isList({ @@ -2816,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); @@ -2835,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); @@ -2854,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); @@ -2881,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) { + if (args[0]->type() == nFloat || args[1]->type() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2461ebc99..845a1ed1b 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6d93e1dc2..6e7ddde8e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,25 +85,25 @@ static void fetchTree( state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->normalType() == nPath || attr.value->normalType() == nString) + if (attr.value->type() == nPath || attr.value->type() == nString) addURI( state, attrs, attr.name, state.coerceToString(*attr.pos, *attr.value, context, false, false) ); - else if (attr.value->normalType() == nString) + else if (attr.value->type() == nString) addURI(state, attrs, attr.name, attr.value->string.s); - else if (attr.value->normalType() == nBool) + else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->normalType() == nInt) + else if (attr.value->type() == nInt) attrs.emplace(attr.name, attr.value->integer); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", @@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->normalType() == nAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index b5f4c8654..bfea24d40 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,7 +16,7 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: out.write(v.integer); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 26be07cff..7464455d8 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -58,7 +58,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->normalType() == nString) + if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->normalType() == nString) + if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 8b312bf03..61ea1d64b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -136,12 +136,12 @@ public: // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type - // normalType() == nThunk + // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; inline bool isBlackhole() const { return internalType == tBlackhole; }; - // normalType() == nFunction + // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; @@ -204,7 +204,7 @@ public: // Returns the normal type of a Value. This only returns nThunk if the // Value hasn't been forceValue'd - inline ValueType normalType() const + inline ValueType type() const { switch (internalType) { case tInt: return nInt; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 404fd5111..6c2e075ed 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) i.queryName(), j) }); else { - if (v->normalType() == nString) { + if (v->type() == nString) { attrs2["type"] = "string"; attrs2["value"] = v->string.s; xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nInt) { + } else if (v->type() == nInt) { attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nFloat) { + } else if (v->type() == nFloat) { attrs2["type"] = "float"; attrs2["value"] = (format("%1%") % v->fpoint).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nBool) { + } else if (v->type() == nBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->normalType() == nList) { + } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->normalType() != nString) continue; + if (v->listElems()[j]->type() != nString) continue; XMLAttrs attrs3; attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } - } else if (v->normalType() == nAttrs) { + } else if (v->type() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); Bindings & attrs = *v->attrs; for (auto &i : attrs) { Attr & a(*attrs.find(i.name)); - if(a.value->normalType() != nString) continue; + if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = i.name; attrs3["value"] = a.value->string.s; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index bba3b1bc6..ea82e5300 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand recurse = [&](Value & v, const Pos & pos, const Path & path) { state->forceValue(v); - if (v.normalType() == nString) + if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path, v.string.s); - else if (v.normalType() == nAttrs) { + else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e4da0348c..066430c5d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand if (v.isLambda()) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.normalType() == nAttrs) { + } else if (v.type() == nAttrs) { for (auto & attr : *v.attrs) try { state->forceValue(*attr.value, *attr.pos); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 047e2dc59..673155078 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -446,7 +446,7 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.normalType() == nPath || v.normalType() == nString) { + if (v.type() == nPath || v.type() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); @@ -669,7 +669,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.normalType()) { + switch (v.type()) { case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; From b70d22baca3e8826392b61aa53955c6da74b8724 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 18 Dec 2020 14:38:49 +0100 Subject: [PATCH 022/113] Replace Value type setters with mk* functions Move clearValue inside Value mkInt instead of setInt mkBool instead of setBool mkString instead of setString mkPath instead of setPath mkNull instead of setNull mkAttrs instead of setAttrs mkList instead of setList* mkThunk instead of setThunk mkApp instead of setApp mkLambda instead of setLambda mkBlackhole instead of setBlackhole mkPrimOp instead of setPrimOp mkPrimOpApp instead of setPrimOpApp mkExternal instead of setExternal mkFloat instead of setFloat Add note that the static mk* function should be removed eventually --- src/libexpr/attr-set.cc | 4 +- src/libexpr/eval-inline.hh | 6 +- src/libexpr/eval.cc | 49 +++------- src/libexpr/nixexpr.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/value.hh | 193 ++++++++++++++++++++++++------------- src/nix/repl.cc | 4 +- 7 files changed, 144 insertions(+), 116 deletions(-) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 17886a426..b6091c955 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) v = vEmptySet; return; } - clearValue(v); - v.setAttrs(); - v.attrs = allocBindings(capacity); + v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; } diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index e56ce261c..f6dead6b0 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -36,13 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos) Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.setBlackhole(); + v.mkBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.setThunk(); - v.thunk.env = env; - v.thunk.expr = expr; + v.mkThunk(env, expr); throw; } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2f8d6d259..5a641d02c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -430,9 +430,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) } } - clearValue(vEmptySet); - vEmptySet.setAttrs(); - vEmptySet.attrs = allocBindings(0); + vEmptySet.mkAttrs(allocBindings(0)); createBaseEnv(); } @@ -548,16 +546,14 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->setPrimOp(); - vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; + vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } Value * v = allocValue(); - v->setPrimOp(); - v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; + v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -572,8 +568,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->setPrimOp(); - vPrimOp->primOp = new PrimOp(std::move(primOp)); + vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(primOp.name, v); @@ -584,8 +579,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) primOp.name = symbols.create(std::string(primOp.name, 2)); Value * v = allocValue(); - v->setPrimOp(); - v->primOp = new PrimOp(std::move(primOp)); + v->mkPrimOp(new PrimOp(std::move(primOp))); staticBaseEnv.vars[envName] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); @@ -708,15 +702,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con void mkString(Value & v, const char * s) { - mkStringNoCopy(v, dupString(s)); + v.mkString(dupString(s)); } Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.setString(); - v.string.s = dupStringWithLen(s.data(), s.size()); - v.string.context = 0; + v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -731,7 +723,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) void mkPath(Value & v, const char * s) { - mkPathNoCopy(v, dupString(s)); + v.mkPath(dupString(s)); } @@ -792,16 +784,9 @@ Env & EvalState::allocEnv(size_t size) void EvalState::mkList(Value & v, size_t size) { - clearValue(v); - if (size == 1) - v.setList1(); - else if (size == 2) - v.setList2(); - else { - v.setListN(); - v.bigList.size = size; - v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; - } + v.mkList(size); + if (size > 2) + v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *)); nrListElems += size; } @@ -810,9 +795,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.setThunk(); - v.thunk.env = &env; - v.thunk.expr = expr; + v.mkThunk(&env, expr); nrThunks++; } @@ -1207,9 +1190,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.setLambda(); - v.lambda.env = &env; - v.lambda.fun = this; + v.mkLambda(&env, this); } @@ -1252,9 +1233,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.setPrimOpApp(); - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; + v.mkPrimOpApp(fun2, &arg); } } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..b80a7de4e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -129,7 +129,7 @@ struct ExprPath : Expr { string s; Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4106f1ec8..45066e9cf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1434,7 +1434,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - mkStringNoCopy(*ent_val, + ent_val->mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 61ea1d64b..b317c1898 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -114,24 +114,6 @@ friend void printValue(std::ostream & str, std::set & active, con public: - inline void setInt() { internalType = tInt; }; - inline void setBool() { internalType = tBool; }; - inline void setString() { internalType = tString; }; - inline void setPath() { internalType = tPath; }; - inline void setNull() { internalType = tNull; }; - inline void setAttrs() { internalType = tAttrs; }; - inline void setList1() { internalType = tList1; }; - inline void setList2() { internalType = tList2; }; - inline void setListN() { internalType = tListN; }; - inline void setThunk() { internalType = tThunk; }; - inline void setApp() { internalType = tApp; }; - inline void setLambda() { internalType = tLambda; }; - inline void setBlackhole() { internalType = tBlackhole; }; - inline void setPrimOp() { internalType = tPrimOp; }; - inline void setPrimOpApp() { internalType = tPrimOpApp; }; - inline void setExternal() { internalType = tExternal; }; - inline void setFloat() { internalType = tFloat; }; - // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type @@ -222,6 +204,123 @@ public: abort(); } + /* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ + inline void clearValue() + { + app.left = app.right = 0; + } + + inline void mkInt(NixInt n) + { + clearValue(); + internalType = tInt; + integer = n; + } + + inline void mkBool(bool b) + { + clearValue(); + internalType = tBool; + boolean = b; + } + + inline void mkString(const char * s, const char * * context = 0) + { + internalType = tString; + string.s = s; + string.context = context; + } + + inline void mkPath(const char * s) + { + clearValue(); + internalType = tPath; + path = s; + } + + inline void mkNull() + { + clearValue(); + internalType = tNull; + } + + inline void mkAttrs(Bindings * a) + { + clearValue(); + internalType = tAttrs; + attrs = a; + } + + inline void mkList(size_t size) + { + clearValue(); + if (size == 1) + internalType = tList1; + else if (size == 2) + internalType = tList2; + else { + internalType = tListN; + bigList.size = size; + } + } + + inline void mkThunk(Env * e, Expr * ex) + { + internalType = tThunk; + thunk.env = e; + thunk.expr = ex; + } + + inline void mkApp(Value * l, Value * r) + { + internalType = tApp; + app.left = l; + app.right = r; + } + + inline void mkLambda(Env * e, ExprLambda * f) + { + internalType = tLambda; + lambda.env = e; + lambda.fun = f; + } + + inline void mkBlackhole() + { + internalType = tBlackhole; + // Value will be overridden anyways + } + + inline void mkPrimOp(PrimOp * p) + { + clearValue(); + internalType = tPrimOp; + primOp = p; + } + + + inline void mkPrimOpApp(Value * l, Value * r) + { + internalType = tPrimOpApp; + app.left = l; + app.right = r; + } + + inline void mkExternal(ExternalValueBase * e) + { + clearValue(); + internalType = tExternal; + external = e; + } + + inline void mkFloat(NixFloat n) + { + clearValue(); + internalType = tFloat; + fpoint = n; + } + bool isList() const { return internalType == tList1 || internalType == tList2 || internalType == tListN; @@ -251,86 +350,42 @@ public: }; -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.left = v.app.right = 0; -} - +// TODO: Remove these static functions, replace call sites with v.mk* instead static inline void mkInt(Value & v, NixInt n) { - clearValue(v); - v.setInt(); - v.integer = n; + v.mkInt(n); } - static inline void mkFloat(Value & v, NixFloat n) { - clearValue(v); - v.setFloat(); - v.fpoint = n; + v.mkFloat(n); } - static inline void mkBool(Value & v, bool b) { - clearValue(v); - v.setBool(); - v.boolean = b; + v.mkBool(b); } - static inline void mkNull(Value & v) { - clearValue(v); - v.setNull(); + v.mkNull(); } - static inline void mkApp(Value & v, Value & left, Value & right) { - v.setApp(); - v.app.left = &left; - v.app.right = &right; + v.mkApp(&left, &right); } - -static inline void mkPrimOpApp(Value & v, Value & left, Value & right) -{ - v.setPrimOpApp(); - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.setString(); - v.string.s = s; - v.string.context = 0; -} - - static inline void mkString(Value & v, const Symbol & s) { - mkStringNoCopy(v, ((const string &) s).c_str()); + v.mkString(((const string &) s).c_str()); } void mkString(Value & v, const char * s); -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.setPath(); - v.path = s; -} - - void mkPath(Value & v, const char * s); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 673155078..a992d8732 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -551,9 +551,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.setThunk(); - v.thunk.env = env; - v.thunk.expr = e; + v.mkThunk(env, e); addVarToScope(state->symbols.create(name), v); } else { Value v; From 1a1af75338cb9ed28dc00de2e696d8efc5d37287 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 15:33:12 +0000 Subject: [PATCH 023/113] Overhaul store subclassing We embrace virtual the rest of the way, and get rid of the `assert(false)` 0-param constructors. We also list config base classes first, so the constructor order is always: 1. all the configs 2. all the stores Each in the same order --- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build/derivation-goal.cc | 11 ++++++++--- src/libstore/dummy-store.cc | 3 ++- src/libstore/http-binary-cache-store.cc | 5 ++++- src/libstore/legacy-ssh-store.cc | 3 ++- src/libstore/local-binary-cache-store.cc | 5 ++++- src/libstore/local-fs-store.hh | 2 +- src/libstore/local-store.cc | 2 ++ src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 11 ++++++++++- src/libstore/s3-binary-cache-store.hh | 6 ++---- src/libstore/ssh-store.cc | 4 +++- src/libstore/store-api.hh | 20 +------------------- src/libstore/uds-remote-store.cc | 3 +++ src/libstore/uds-remote-store.hh | 7 +------ 17 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 07a8b2beb..443a53cac 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "enable multi-threading compression, available for xz only currently"}; }; -class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store { private: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f494545fb..47d11dc53 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1985,7 +1985,7 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } -struct RestrictedStoreConfig : LocalFSStoreConfig +struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; const std::string name() { return "Restricted Store"; } @@ -1994,14 +1994,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore { ref next; DerivationGoal & goal; RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) { } Path getRealStoreDir() override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 91fc178db..3c7caf8f2 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig { const std::string name() override { return "Dummy Store"; } }; -struct DummyStore : public Store, public virtual DummyStoreConfig +struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(const std::string scheme, const std::string uri, const Params & params) : DummyStore(params) @@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) + , DummyStoreConfig(params) , Store(params) { } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d2a89f96..0a3afcd51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Http Binary Cache Store"; } }; -class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig +class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -36,6 +36,9 @@ public: const Path & _cacheUri, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , cacheUri(scheme + "://" + _cacheUri) { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ad1779aea..253c0033e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig const std::string name() override { return "Legacy SSH Store"; } }; -struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in @@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig LegacySSHStore(const string & scheme, const string & host, const Params & params) : StoreConfig(params) + , LegacySSHStoreConfig(params) , Store(params) , host(host) , connections(make_ref>( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index bb7464989..a58b7733f 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Local Binary Cache Store"; } }; -class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig +class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -24,6 +24,9 @@ public: const Path & binaryCacheDir, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8eccd8236..55941b771 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "log", "directory where Nix will store state"}; }; -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store { public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9f9bde4d..c52d4b62a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -100,6 +100,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) LocalStore::LocalStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 877dba742..ae9497b2e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public LocalFSStore, public virtual LocalStoreConfig +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore { private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f1f4d0516..be07f02dc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) + : RemoteStoreConfig(params) + , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fdd53e6ed..b3a9910a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store { public: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d6edafd7e..6bfbee044 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } +S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) + : BinaryCacheStoreConfig(params) + , BinaryCacheStore(params) +{ } + struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; @@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "S3 Binary Cache Store"; } }; -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { std::string bucketName; @@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache const std::string & bucketName, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(params) + , Store(params) + , BinaryCacheStore(params) , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..bce828b11 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -6,13 +6,11 @@ namespace nix { -class S3BinaryCacheStore : public BinaryCacheStore +class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: - S3BinaryCacheStore(const Params & params) - : BinaryCacheStore(params) - { } + S3BinaryCacheStore(const Params & params); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 08d0bd565..17c258201 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig const std::string name() override { return "SSH Store"; } }; -class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig +class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , RemoteStoreConfig(params) + , SSHStoreConfig(params) , Store(params) , RemoteStore(params) , host(host) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ce95b78b1..9bcff08eb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -175,25 +175,7 @@ struct StoreConfig : public Config { using Config::Config; - /** - * When constructing a store implementation, we pass in a map `params` of - * parameters that's supposed to initialize the associated config. - * To do that, we must use the `StoreConfig(StringMap & params)` - * constructor, so we'd like to `delete` its default constructor to enforce - * it. - * - * However, actually deleting it means that all the subclasses of - * `StoreConfig` will have their default constructor deleted (because it's - * supposed to call the deleted default constructor of `StoreConfig`). But - * because we're always using virtual inheritance, the constructors of - * child classes will never implicitely call this one, so deleting it will - * be more painful than anything else. - * - * So we `assert(false)` here to ensure at runtime that the right - * constructor is always called without having to redefine a custom - * constructor for each `*Config` class. - */ - StoreConfig() { assert(false); } + StoreConfig() = delete; virtual ~StoreConfig() { } diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 24f3e9c6d..cac4fa036 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -15,6 +15,9 @@ namespace nix { UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , UDSRemoteStoreConfig(params) , Store(params) , LocalFSStore(params) , RemoteStore(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index e5de104c9..ddc7716cd 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon { } - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - const std::string name() override { return "Local Daemon Store"; } }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore { public: From 346baec783a7423aa5b6cacaf7eebb8d22d4ce79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Dec 2020 13:04:24 +0100 Subject: [PATCH 024/113] Move doc() to Args --- src/libutil/args.cc | 4 ++-- src/libutil/args.hh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 61f9503ec..a929ea5ac 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -254,6 +254,8 @@ nlohmann::json Args::toJSON() res["description"] = description(); res["flags"] = std::move(flags); res["args"] = std::move(args); + auto s = doc(); + if (s != "") res.emplace("doc", stripIndentation(s)); return res; } @@ -378,8 +380,6 @@ nlohmann::json Command::toJSON() auto res = Args::toJSON(); res["examples"] = std::move(exs); - auto s = doc(); - if (s != "") res.emplace("doc", stripIndentation(s)); return res; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 8069fd70f..68bbbb4f7 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -25,6 +25,9 @@ public: /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } + /* Return documentation about this command, in Markdown format. */ + virtual std::string doc() { return ""; } + protected: static const size_t ArityAny = std::numeric_limits::max(); @@ -225,9 +228,6 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; - /* Return documentation about this command, in Markdown format. */ - virtual std::string doc() { return ""; } - struct Example { std::string description; From ae7351dbeed3d03087b6b56d23b4b3942de1507d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 14:19:36 +0100 Subject: [PATCH 025/113] Add 'nix build' manpage --- src/nix/build.cc | 19 +++------- src/nix/build.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 src/nix/build.md diff --git a/src/nix/build.cc b/src/nix/build.cc index 67be4024b..c2974d983 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -43,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile return "build a derivation or fetch a store path"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To build and run GNU Hello from NixOS 17.03:", - "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" - }, - Example{ - "To build the build.x86_64-linux attribute from release.nix:", - "nix build -f release.nix build.x86_64-linux" - }, - Example{ - "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs#hello" - }, - }; + return + #include "build.md" + ; } void run(ref store) override diff --git a/src/nix/build.md b/src/nix/build.md new file mode 100644 index 000000000..c2f3e387a --- /dev/null +++ b/src/nix/build.md @@ -0,0 +1,92 @@ +R""( + +# Examples + +* Build the default package from the flake in the current directory: + + ```console + # nix build + ``` + +* Build and run GNU Hello from the `nixpkgs` flake: + + ```console + # nix build nixpkgs#hello + # ./result/bin/hello + Hello, world! + ``` + +* Build GNU Hello and Cowsay, leaving two result symlinks: + + ```console + # nix build nixpkgs#hello nixpkgs#cowsay + # ls -l result* + lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +* Build a specific output: + + ```console + # nix build nixpkgs#glibc.dev + # ls -ld ./result-dev + lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev + ``` + +* Build attribute `build.x86_64-linux` from (non-flake) Nix expression + `release.nix`: + + ```console + # nix build -f release.nix build.x86_64-linux + ``` + +* Build a NixOS system configuration from a flake, and make a profile + point to the result: + + ```console + # nix build --profile /nix/var/nix/profiles/system \ + ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel + ``` + + (This is essentially what `nixos-rebuild` does.) + +* Build an expression specified on the command line: + + ```console + # nix build --impure --expr \ + 'with import {}; + runCommand "foo" { + buildInputs = [ hello ]; + } + "hello > $out"' + # cat ./result + Hello, world! + ``` + + Note that `--impure` is needed because we're using ``, + which relies on the `$NIX_PATH` environment variable. + +* Fetch a store path from the configured substituters, if it doesn't + already exist: + + ```console + # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +# Description + +`nix build` builds the specified *installables*. Installables that +resolve to derivations are built (or substituted if possible). Store +path installables are substituted. + +Unless `--no-link` is specified, after a successful build, it creates +symlinks to the store paths of the installables. These symlinks have +the prefix `./result` by default; this can be overriden using the +`--out-link` option. Each symlink has a suffix `--`, where +*N* is the index of the installable (with the left-most installable +having index 0), and *outname* is the symbolic derivation output name +(e.g. `bin`, `dev` or `lib`). `-` is omitted if *N* = 0, and +`-` is omitted if *outname* = `out` (denoting the default +output). + +)"" From 09660b855778531be14968b720308d092af4dd2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:16:23 +0100 Subject: [PATCH 026/113] Add 'nix run' and 'nix shell' manpages --- src/nix/run.cc | 42 +++++------------------ src/nix/run.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ src/nix/shell.md | 48 ++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 34 deletions(-) create mode 100644 src/nix/run.md create mode 100644 src/nix/shell.md diff --git a/src/nix/run.cc b/src/nix/run.cc index 92a52c6cd..c2400f9ee 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -86,26 +86,11 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment return "run a shell in which the specified packages are available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To start a shell providing GNU Hello from NixOS 20.03:", - "nix shell nixpkgs/nixos-20.03#hello" - }, - Example{ - "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs#youtube-dl" - }, - Example{ - "To run GNU Hello:", - "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'" - }, - Example{ - "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs#hello -c hello" - }, - }; + return + #include "shell.md" + ; } void run(ref store) override @@ -168,22 +153,11 @@ struct CmdRun : InstallableCommand, RunCommon return "run a Nix application"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To run Blender:", - "nix run blender-bin" - }, - Example{ - "To run vim from nixpkgs:", - "nix run nixpkgs#vim" - }, - Example{ - "To run vim from nixpkgs with arguments:", - "nix run nixpkgs#vim -- --help" - }, - }; + return + #include "run.md" + ; } Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/run.md b/src/nix/run.md new file mode 100644 index 000000000..c178e8b13 --- /dev/null +++ b/src/nix/run.md @@ -0,0 +1,87 @@ +R""( + +# Examples + +* Run the default app from the `blender-bin` flake: + + ```console + # nix run blender-bin + ``` + +* Run a non-default app from the `blender-bin` flake: + + ```console + # nix run blender-bin#blender_2_83 + ``` + + Tip: you can find apps provided by this flake by running `nix flake + show blender-bin`. + +* Run `vim` from the `nixpkgs` flake: + + ```console + # nix run nixpkgs#vim + ``` + + Note that `vim` (as of the time of writing of this page) is not an + app but a package. Thus, Nix runs the eponymous file from the `vim` + package. + +* Run `vim` with arguments: + + ```console + # nix run nixpkgs#vim -- --help + ``` + +# Description + +`nix run` builds and runs *installable*, which must evaluate to an +*app* or a regular Nix derivation. + +If *installable* evaluates to an *app* (see below), it executes the +program specified by the app definition. + +If *installable* evaluates to a derivation, it will try to execute the +program `/bin/`, where *out* is the primary output store +path of the derivation and *name* is the name part of the value of the +`name` attribute of the derivation (e.g. if `name` is set to +`hello-1.10`, it will run `$out/bin/hello`). + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `defaultApp.` + +* `defaultPackage.` + +If an attribute *name* is given, `nix run` tries the following flake +output attributes: + +* `apps..` + +* `packages..` + +* `legacyPackages..` + +# Apps + +An app is specified by a flake output attribute named +`apps..` or `defaultApp.`. It looks like this: + +```nix +apps.x86_64-linux.blender_2_79 = { + type = "app"; + program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; +}; +``` + +The only supported attributes are: + +* `type` (required): Must be set to `app`. + +* `program` (required): The full path of the executable to run. It + must reside in the Nix store. + +)"" diff --git a/src/nix/shell.md b/src/nix/shell.md new file mode 100644 index 000000000..2a379e03f --- /dev/null +++ b/src/nix/shell.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Start a shell providing `youtube-dl` from the `nixpkgs` flake: + + ```console + # nix shell nixpkgs#youtube-dl + # youtube-dl --version + 2020.11.01.1 + ``` + +* Start a shell providing GNU Hello from NixOS 20.03: + + ```console + # nix shell nixpkgs/nixos-20.03#hello + ``` + +* Run GNU Hello: + + ```console + # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + Hi everybody! + ``` + +* Run GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello -c hello + ``` + +* Start a shell providing GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + ``` + + Note that it's necessary to specify `bash` explicitly because your + default shell (e.g. `/bin/bash`) generally will not exist in the + chroot. + +# Description + +`nix shell` runs a command in an environment in which the `$PATH` +variable provides the specified *installables*. If not command is +specified, it starts the default shell of your user account. + +)"" From 9dcd0aebc59a53c622d709c33d4c6f5e20bc0ac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:34:45 +0100 Subject: [PATCH 027/113] generate-manpage.nix: Fix short names --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fbd7f3e7d..9f30c8fbc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -43,7 +43,7 @@ let if flag.category or "" != "config" then " - `--${longName}`" - + (if flag ? shortName then " / `${flag.shortName}`" else "") + + (if flag ? shortName then " / `-${flag.shortName}`" else "") + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") + " \n" + " " + flag.description + "\n\n" From 28ee307fd8d38e8f6c8dd9d2575435036f3612cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 17:49:58 +0100 Subject: [PATCH 028/113] Add 'nix copy' manpage --- src/nix/copy.cc | 29 ++++--------------------- src/nix/copy.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 src/nix/copy.md diff --git a/src/nix/copy.cc b/src/nix/copy.cc index cb31aac8f..2394eb46d 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -54,32 +54,11 @@ struct CmdCopy : StorePathsCommand return "copy paths between Nix stores"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache $(type -p firefox)" - }, - Example{ - "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server /run/current-system" - }, - Example{ - "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" - }, -#ifdef ENABLE_S3 - Example{ - "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello" - }, - Example{ - "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello" - }, -#endif - }; + return + #include "copy.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/copy.md b/src/nix/copy.md new file mode 100644 index 000000000..25e0ddadc --- /dev/null +++ b/src/nix/copy.md @@ -0,0 +1,58 @@ +R""( + +# Examples + +* Copy Firefox from the local store to a binary cache in `/tmp/cache`: + + ```console + # nix copy --to file:///tmp/cache $(type -p firefox) + ``` + + Note the `file://` - without this, the destination is a chroot + store, not a binary cache. + +* Copy the entire current NixOS system closure to another machine via + SSH: + + ```console + # nix copy -s --to ssh://server /run/current-system + ``` + + The `-s` flag causes the remote machine to try to substitute missing + store paths, which may be faster if the link between the local and + remote machines is slower than the link between the remote machine + and its substituters (e.g. `https://cache.nixos.org`). + +* Copy a closure from another machine via SSH: + + ```console + # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2 + ``` + +* Copy Hello to a binary cache in an Amazon S3 bucket: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello + ``` + + or to an S3-compatible storage system: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello + ``` + + Note that this only works if Nix is built with AWS support. + +* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`: + + ```console + # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs + ``` + +# Description + +`nix copy` copies store path closures between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)"" From e9de689a6efca961099f1acfc574009f31a6f130 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 18:09:30 +0100 Subject: [PATCH 029/113] Add 'nix search' manpage --- src/nix/search.cc | 25 ++++------------ src/nix/search.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 20 deletions(-) create mode 100644 src/nix/search.md diff --git a/src/nix/search.cc b/src/nix/search.cc index 47770e128..9f864b3a4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -41,29 +41,14 @@ struct CmdSearch : InstallableCommand, MixJSON std::string description() override { - return "query available packages"; + return "search for packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show all packages in the flake in the current directory:", - "nix search" - }, - Example{ - "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:", - "nix search nixpkgs blender" - }, - Example{ - "To search for Firefox or Chromium:", - "nix search nixpkgs 'firefox|chromium'" - }, - Example{ - "To search for packages containing 'git' and either 'frontend' or 'gui':", - "nix search nixpkgs git 'frontend|gui'" - } - }; + return + #include "search.md" + ; } Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/search.md b/src/nix/search.md new file mode 100644 index 000000000..d182788a6 --- /dev/null +++ b/src/nix/search.md @@ -0,0 +1,72 @@ +R""( + +# Examples + +* Show all packages in the `nixpkgs` flake: + + ```console + # nix search nixpkgs + * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) + A set of ambisonics ladspa plugins + + * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0) + Application with primary purpose of idling Steam cards from multiple accounts simultaneously + … + ``` + +* Show packages in the `nixpkgs` flake containing `blender` in its + name or description: + + ```console + # nix search nixpkgs blender + * legacyPackages.x86_64-linux.blender (2.91.0) + 3D Creation/Animation/Publishing System + ``` + +* Search for packages underneath the attribute `gnome3` in Nixpkgs: + + ```console + # nix search nixpkgs#gnome3 vala + * legacyPackages.x86_64-linux.gnome3.vala (0.48.9) + Compiler for GObject type system + ``` + +* Show all packages in the flake in the current directory: + + ```console + # nix search + ``` + +* Search for Firefox or Chromium: + + ```console + # nix search nixpkgs 'firefox|chromium' + ``` + +* Search for packages containing `git'`and either `frontend` or `gui`: + + ```console + # nix search nixpkgs git 'frontend|gui' + ``` + +# Description + +`nix search` searches *installable* (which must be evaluatable, e.g. a +flake) for packages whose name or description matches all of the +regular expressions *regex*. For each matching package, It prints the +full attribute name (from the root of the installable), the version +and the `meta.description` field, highlighting the substrings that +were matched by the regular expressions. If no regular expressions are +specified, all packages are shown. + +# Flake output attributes + +If no flake output attribute is given, `nix search` searches for +packages: + +* Directly underneath `packages.`. + +* Underneath `legacyPackages.`, recursing into attribute sets + that contain an attribute `recurseForDerivations = true`. + +)"" From 42cc98f8d66627ff7e396fb809034d3389b3bd0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 21:52:22 +0100 Subject: [PATCH 030/113] Add 'nix develop' and `nix print-dev-env' manpages --- src/nix/develop.cc | 38 ++++------------ src/nix/develop.md | 94 ++++++++++++++++++++++++++++++++++++++++ src/nix/print-dev-env.md | 19 ++++++++ 3 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 src/nix/develop.md create mode 100644 src/nix/print-dev-env.md diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 457d94382..edd87f246 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -385,30 +385,11 @@ struct CmdDevelop : Common, MixEnvironment return "run a bash shell that provides the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build environment of GNU hello:", - "nix develop nixpkgs#hello" - }, - Example{ - "To get the build environment of the default package of flake in the current directory:", - "nix develop" - }, - Example{ - "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs#hello" - }, - Example{ - "To use a build environment previously recorded in a profile:", - "nix develop /tmp/my-shell" - }, - Example{ - "To replace all occurences of a store path with a writable directory:", - "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev" - }, - }; + return + #include "develop.md" + ; } void run(ref store) override @@ -495,14 +476,11 @@ struct CmdPrintDevEnv : Common return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs#hello)" - }, - }; + return + #include "print-dev-env.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/develop.md b/src/nix/develop.md new file mode 100644 index 000000000..7a906d10d --- /dev/null +++ b/src/nix/develop.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Start a shell with the build environment of the default package of + the flake in the current directory: + + ```console + # nix develop + ``` + + Typical commands to run inside this shell are: + + ```console + # configurePhase + # buildPhase + # installPhase + ``` + + Alternatively, you can run whatever build tools your project uses + directly, e.g. for a typical Unix project: + + ```console + # ./configure --prefix=$out + # make + # make install + ``` + +* Run a particular build phase directly: + + ```console + # nix develop --configure + # nix develop --build + # nix develop --check + # nix develop --install + # nix develop --installcheck + ``` + +* Start a shell with the build environment of GNU Hello: + + ```console + # nix develop nixpkgs#hello + ``` + +* Record a build environment in a profile: + + ```console + # nix develop --profile /tmp/my-build-env nixpkgs#hello + ``` + +* Use a build environment previously recorded in a profile: + + ```cosnole + # nix develop /tmp/my-build-env + ``` + +* Replace all occurences of the store path corresponding to + `glibc.dev` with a writable directory: + + ```console + # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev + ``` + + Note that this is useful if you're running a `nix develop` shell for + `nixpkgs#glibc` in `~/my-glibc` and want to compile another package + against it. + +# Description + +`nix develop` starts a `bash` shell that provides an interactive build +environment nearly identical to what Nix would use to build +*installable*. Inside this shell, environment variables and shell +functions are set up so that you can interactively and incrementally +build your package. + +Nix determines the build environment by building a modified version of +the derivation *installable* that just records the environment +initialised by `stdenv` and exits. This build environment can be +recorded into a profile using `--profile`. + +The prompt used by the `bash` shell can be customised by setting the +`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in +the flake's `nixConfig` attribute. + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `devShell.` + +* `defaultPackage.` + +)"" diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md new file mode 100644 index 000000000..b80252acf --- /dev/null +++ b/src/nix/print-dev-env.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Apply the build environment of GNU hello to the current shell: + + ```console + # . <(nix print-dev-env nixpkgs#hello) + ``` + +# Description + +This command prints a shell script that can be sourced by `b`ash and +that sets the environment variables and shell functions defined by the +build process of *installable*. This allows you to get a similar build +environment in your current shell rather than in a subshell (as with +`nix develop`). + +)"" From e90e74523238f37748d9f406732919374d7ee561 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 22:57:14 +0100 Subject: [PATCH 031/113] Add 'nix registry' manpages This also documents the registry format and matching/unification semantics (though not quite correctly). --- src/nix/registry-add.md | 33 +++++++++++++ src/nix/registry-list.md | 29 +++++++++++ src/nix/registry-pin.md | 38 +++++++++++++++ src/nix/registry-remove.md | 16 +++++++ src/nix/registry.cc | 35 ++++++++++++++ src/nix/registry.md | 98 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+) create mode 100644 src/nix/registry-add.md create mode 100644 src/nix/registry-list.md create mode 100644 src/nix/registry-pin.md create mode 100644 src/nix/registry-remove.md create mode 100644 src/nix/registry.md diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md new file mode 100644 index 000000000..80a31996a --- /dev/null +++ b/src/nix/registry-add.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03 + ``` + +* Pin `nixpkgs` to a specific revision: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + +* Add an entry that redirects a specific branch of `nixpkgs` to + another fork: + + ```console + # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *from-url* to flake reference *to-url*. If an entry for +*from-url* already exists, it is overwritten. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md new file mode 100644 index 000000000..30b6e29d8 --- /dev/null +++ b/src/nix/registry-list.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Show the contents of all registries: + + ```console + # nix registry list + user flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5 + system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124 + global flake:blender-bin github:edolstra/nix-warez?dir=blender + global flake:dwarffs github:edolstra/dwarffs + … + ``` + +# Description + +This command displays the contents of all registries on standard +output. Each line represents one registry entry in the format *type* +*from* *to*, where *type* denotes the registry containing the entry: + +* `flags`: entries specified on the command line using `--override-flake`. +* `user`: the user registry. +* `system`: the system registry. +* `global`: the global registry. + +See the [`nix registry` manual page](./nix3-registry.md) for more details. + +)"" diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md new file mode 100644 index 000000000..6e97e003e --- /dev/null +++ b/src/nix/registry-pin.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Pin `nixpkgs` to its most recent Git revision: + + ```console + # nix registry pin nixpkgs + ``` + + Afterwards the user registry will have an entry like this: + + ```console + nix registry list | grep '^user ' + user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + + and `nix flake info` will say: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + … + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *url* to the corresponding *locked* flake reference, that +is, a flake reference that specifies an exact revision or content +hash. This ensures that until this registry entry is removed, all uses +of *url* will resolve to exactly the same flake. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md new file mode 100644 index 000000000..4c0eb4947 --- /dev/null +++ b/src/nix/registry-remove.md @@ -0,0 +1,16 @@ +R""( + +# Examples + +* Remove the entry `nixpkgs` from the user registry: + + ```console + # nix registry remove nixpkgs + ``` + +# Description + +This command removes from the user registry any entry for flake +reference *url*. + +)"" diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 9352e00a7..f9719600f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -17,6 +17,13 @@ struct CmdRegistryList : StoreCommand return "list available Nix flakes"; } + std::string doc() override + { + return + #include "registry-list.md" + ; + } + void run(nix::ref store) override { using namespace fetchers; @@ -47,6 +54,13 @@ struct CmdRegistryAdd : MixEvalArgs, Command return "add/replace flake in user flake registry"; } + std::string doc() override + { + return + #include "registry-add.md" + ; + } + CmdRegistryAdd() { expectArg("from-url", &fromUrl); @@ -75,6 +89,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command return "remove flake from user flake registry"; } + std::string doc() override + { + return + #include "registry-remove.md" + ; + } + CmdRegistryRemove() { expectArg("url", &url); @@ -97,6 +118,13 @@ struct CmdRegistryPin : virtual Args, EvalCommand return "pin a flake to its current version in user flake registry"; } + std::string doc() override + { + return + #include "registry-pin.md" + ; + } + CmdRegistryPin() { expectArg("url", &url); @@ -132,6 +160,13 @@ struct CmdRegistry : virtual NixMultiCommand return "manage the flake registry"; } + std::string doc() override + { + return + #include "registry.md" + ; + } + Category category() override { return catSecondary; } void run() override diff --git a/src/nix/registry.md b/src/nix/registry.md new file mode 100644 index 000000000..557e5795b --- /dev/null +++ b/src/nix/registry.md @@ -0,0 +1,98 @@ +R""( + +# Description + +`nix flake` provides subcommands for managing *flake +registries*. Flake registries are a convenience feature that allows +you to refer to flakes using symbolic identifiers such as `nixpkgs`, +rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You +can use these identifiers on the command line (e.g. when you do `nix +run nixpkgs#hello`) or in flake input specifications in `flake.nix` +files. The latter are automatically resolved to full URLs and recorded +in the flake's `flake.lock` file. + +In addition, the flake registry allows you to redirect arbitrary flake +references (e.g. `github:NixOS/patchelf`) to another location, such as +a local fork. + +There are multiple registries. These are, in order from lowest to +highest precedence: + +* The global registry, which is a file downloaded from the URL + specified by the setting `flake-registry`. It is cached locally and + updated automatically when it's older than `tarball-ttl` + seconds. The default global registry is kept in [a GitHub + repository](https://github.com/NixOS/flake-registry). + +* The system registry, which is shared by all users. The default + location is `/etc/nix/registry.json`. On NixOS, the system registry + can be specified using the NixOS option `nix.registry`. + +* The user registry `~/.config/nix/registry.json`. This registry can + be modified by commands such as `nix flake pin`. + +* Overrides specified on the command line using the option + `--override-flake`. + +# Registry format + +A registry is a JSON file with the following format: + +```json +{ + "version": 2, + [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "github", + "owner": "NixOS", + "repo": "nixpkgs" + } + }, + ... + ] +} +``` + +That is, it contains a list of objects with attributes `from` and +`to`, both of which contain a flake reference in attribute +representation. (For example, `{"type": "indirect", "id": "nixpkgs"}` +is the attribute representation of `nixpkgs`, while `{"type": +"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute +representation of `github:NixOS/nixpkgs`.) + +Given some flake reference *R*, a registry entry is used if its +`from` flake reference *matches* *R*. *R* is then replaced by the +*unification* of the `to` flake reference with *R*. + +# Matching + +The `from` flake reference in a registry entry *matches* some flake +reference *R* if the attributes in `from` are the same as the +attributes in `R`. For example: + +* `nixpkgs` matches with `nixpkgs`. + +* `nixpkgs` matches with `nixpkgs/nixos-20.09`. + +* `nixpkgs/nixos-20.09` does not match with `nixpkgs`. + +* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`. + +# Unification + +The `to` flake reference in a registry entry is *unified* with some flake +reference *R* by taking `to` and applying the `rev` and `ref` +attributes from *R*, if specified. For example: + +* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`. + +* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +)"" From 4e065229c75455d3cee8bc9e791c10faa93c1b50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Dec 2020 23:01:50 +0100 Subject: [PATCH 032/113] Typo --- src/nix/develop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.md b/src/nix/develop.md index 7a906d10d..e71d9f8aa 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -50,7 +50,7 @@ R""( * Use a build environment previously recorded in a profile: - ```cosnole + ```console # nix develop /tmp/my-build-env ``` From b2262be19babc37a54bed4384fecba11d87f7364 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 12:55:24 +0100 Subject: [PATCH 033/113] Add 'nix edit' manpage --- src/nix/edit.cc | 11 ++++------- src/nix/edit.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 src/nix/edit.md diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 51c16f5a9..6472dd27a 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand return "open the Nix expression of a Nix package in $EDITOR"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs#hello" - }, - }; + return + #include "edit.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/edit.md b/src/nix/edit.md new file mode 100644 index 000000000..80563d06b --- /dev/null +++ b/src/nix/edit.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Open the Nix expression of the GNU Hello package: + + ```console + # nix edit nixpkgs#hello + ``` + +* Get the filename and line number used by `nix edit`: + + ```console + # nix eval --raw nixpkgs#hello.meta.position + /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15 + ``` + +# Description + +This command opens the Nix expression of a derivation in an +editor. The filename and line number of the derivation are taken from +its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets +this attribute to the location of the definition of the +`meta.description`, `version` or `name` derivation attributes. + +The editor to invoke is specified by the `EDITOR` environment +variable. It defaults to `cat`. If the editor is `emacs`, `nano` or +`vim`, it is passed the line number of the derivation using the +argument `+`. + +)"" From 6ce393392b4e303f51a512d64627cbd68a73c0e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 13:07:01 +0100 Subject: [PATCH 034/113] Add 'nix repl' manpage --- src/nix/repl.cc | 12 +++++------ src/nix/repl.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 src/nix/repl.md diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a992d8732..bce8d31dc 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -405,6 +405,7 @@ bool NixRepl::processLine(string line) } if (command == ":?" || command == ":help") { + // FIXME: convert to Markdown, include in the 'nix repl' manpage. std::cout << "The following commands are available:\n" << "\n" @@ -801,14 +802,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "Display all special commands within the REPL:", - "nix repl\nnix-repl> :?" - } - }; + return + #include "repl.md" + ; } void run(ref store) override diff --git a/src/nix/repl.md b/src/nix/repl.md new file mode 100644 index 000000000..bba60f871 --- /dev/null +++ b/src/nix/repl.md @@ -0,0 +1,57 @@ +R""( + +# Examples + +* Display all special commands within the REPL: + + ```console + # nix repl + nix-repl> :? + ``` + +* Evaluate some simple Nix expressions: + + ```console + # nix repl + + nix-repl> 1 + 2 + 3 + + nix-repl> map (x: x * 2) [1 2 3] + [ 2 4 6 ] + ``` + +* Interact with Nixpkgs in the REPL: + + ```console + # nix repl '' + + Loading ''... + Added 12428 variables. + + nix-repl> emacs.name + "emacs-27.1" + + nix-repl> emacs.drvPath + "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" + + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + + nix-repl> :b x + this derivation produced the following outputs: + out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello + + nix-repl> builtins.readFile drv + "Hello, world!\n" + ``` + +# Description + +This command provides an interactive environment for evaluating Nix +expressions. (REPL stands for 'read–eval–print loop'.) + +On startup, it loads the Nix expressions named *files* and adds them +into the lexical scope. You can load addition files using the `:l +` command, or reload all files using `:r`. + +)"" From 58bacc85e79b318f28e456a72be2d6a7c8d86991 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 13:41:47 +0100 Subject: [PATCH 035/113] Add 'nix log' manpage --- src/nix/log.cc | 19 ++++--------------- src/nix/log.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 src/nix/log.md diff --git a/src/nix/log.cc b/src/nix/log.cc index 33a3053f5..67d3742d6 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand return "show the build log of the specified packages or paths, if available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build log of GNU Hello:", - "nix log nixpkgs#hello" - }, - Example{ - "To get the build log of a specific path:", - "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" - }, - Example{ - "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs#hello" - }, - }; + return + #include "log.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/log.md b/src/nix/log.md new file mode 100644 index 000000000..8ee2f1d19 --- /dev/null +++ b/src/nix/log.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Get the build log of GNU Hello: + + ```console + # nix log nixpkgs#hello + ``` + +* Get the build log of a specific store path: + + ```console + # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1 + ``` + +* Get a build log from a specific binary cache: + + ```console + # nix log --store https://cache.nixos.org nixpkgs#hello + ``` + +# Description + +This command prints the log of a previous build of a derivation on +standard output. + +Nix looks for build logs in two places: + +* In the directory `/nix/var/log/nix/drvs`, which contains logs for + locally built derivations. + +* In the binary caches listed in the `substituters` setting. Logs + should be named `/log/`, where + `store-path` is a derivation, + e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`. + For non-derivation store paths, Nix will first try to determine the + deriver by fetching the `.narinfo` file for this store path. + +)"" From f34b1801a4ec9060504a6b712b52763e88b4341f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 17:41:58 +0100 Subject: [PATCH 036/113] Tweak --- src/nix/log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/log.md b/src/nix/log.md index 8ee2f1d19..1c76226a3 100644 --- a/src/nix/log.md +++ b/src/nix/log.md @@ -22,8 +22,8 @@ R""( # Description -This command prints the log of a previous build of a derivation on -standard output. +This command prints the log of a previous build of the derivation +*installable* on standard output. Nix looks for build logs in two places: From 53ce20eab779e0dd156af2ac9ec2318814edc7cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 17:55:59 +0100 Subject: [PATCH 037/113] Add 'nix store ping' manpage --- src/nix/ping-store.cc | 13 +++++-------- src/nix/ping-store.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 src/nix/ping-store.md diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 19b1a55c8..62b645b06 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -8,17 +8,14 @@ struct CmdPingStore : StoreCommand { std::string description() override { - return "test whether a store can be opened"; + return "test whether a store can be accessed"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To test whether connecting to a remote Nix store via SSH works:", - "nix store ping --store ssh://mac1" - }, - }; + return + #include "ping-store.md" + ; } void run(ref store) override diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md new file mode 100644 index 000000000..322093091 --- /dev/null +++ b/src/nix/ping-store.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* Test whether connecting to a remote Nix store via SSH works: + + ```console + # nix store ping --store ssh://mac1 + ``` + +* Test whether a URL is a valid binary cache: + + ```console + # nix store ping --store https://cache.nixos.org + ``` + +* Test whether the Nix daemon is up and running: + + ```console + # nix store ping --store daemon + ``` + +# Description + +This command tests whether a particular Nix store (specified by the +argument `--store` *url*) can be accessed. What this means is +dependent on the type of the store. For instance, for an SSH store it +means that Nix can connect to the specified machine. + +)"" From a407d14339c2c480f0103a501bcd8a3373d935cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 18:34:52 +0100 Subject: [PATCH 038/113] Add 'nix eval' manpage --- src/nix/eval.cc | 27 +++--------------- src/nix/eval.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 src/nix/eval.md diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ea82e5300..321df7495 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -40,30 +40,11 @@ struct CmdEval : MixJSON, InstallableCommand return "evaluate a Nix expression"; } - Examples examples() override + std::string doc() override { - return { - { - "To evaluate a Nix expression given on the command line:", - "nix eval --expr '1 + 2'" - }, - { - "To evaluate a Nix expression from a file or URI:", - "nix eval -f ./my-nixpkgs hello.name" - }, - { - "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs#lib.version" - }, - { - "To print the store path of the Hello package:", - "nix eval --raw nixpkgs#hello" - }, - { - "To get a list of checks in the 'nix' flake:", - "nix eval nix#checks.x86_64-linux --apply builtins.attrNames" - }, - }; + return + #include "eval.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/eval.md b/src/nix/eval.md new file mode 100644 index 000000000..61334cde1 --- /dev/null +++ b/src/nix/eval.md @@ -0,0 +1,74 @@ +R""( + +# Examples + +* Evaluate a Nix expression given on the command line: + + ```console + # nix eval --expr '1 + 2' + ``` + +* Evaluate a Nix expression to JSON: + + ```console + # nix eval --json --expr '{ x = 1; }' + {"x":1} + ``` + +* Evaluate a Nix expression from a file: + + ```console + # nix eval -f ./my-nixpkgs hello.name + ``` + +* Get the current version of the `nixpkgs` flake: + + ```console + # nix eval --raw nixpkgs#lib.version + ``` + +* Print the store path of the Hello package: + + ```console + # nix eval --raw nixpkgs#hello + ``` + +* Get a list of checks in the `nix` flake: + + ```console + # nix eval nix#checks.x86_64-linux --apply builtins.attrNames + ``` + +* Generate a directory with the specified contents: + + ```console + # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }' + # cat ./out/foo + bar + # cat ./out/subdir/bla + 123 + +# Description + +This command evaluates the Nix expression *installable* and prints the +result on standard output. + +# Output format + +`nix eval` can produce output in several formats: + +* By default, the evaluation result is printed as a Nix expression. + +* With `--json`, the evaluation result is printed in JSON format. Note + that this fails if the result contains values that are not + representable as JSON, such as functions. + +* With `--raw`, the evaluation result must be a string, which is + printed verbatim, without any quoting. + +* With `--write-to` *path*, the evaluation result must be a string or + a nested attribute set whose leaf values are strings. These strings + are written to files named *path*/*attrpath*. *path* must not + already exist. + +)"" From 2cc02bbe7675acb4754b928b5c57fa316600e877 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 19:21:48 +0100 Subject: [PATCH 039/113] Add 'nix nar' manpages --- src/nix/cat.cc | 7 +++++++ src/nix/dump-path.cc | 11 ++++------- src/nix/ls.cc | 13 ++++++------- src/nix/nar-cat.md | 19 +++++++++++++++++++ src/nix/nar-dump-path.md | 17 +++++++++++++++++ src/nix/nar-ls.md | 24 ++++++++++++++++++++++++ src/nix/nar.cc | 9 ++++++++- src/nix/nar.md | 13 +++++++++++++ 8 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 src/nix/nar-cat.md create mode 100644 src/nix/nar-dump-path.md create mode 100644 src/nix/nar-ls.md create mode 100644 src/nix/nar.md diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 2ecffc9a5..fe2f0a241 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -62,6 +62,13 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } + std::string doc() override + { + return + #include "nar-cat.md" + ; + } + void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 256db64a9..63393ef9c 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -49,14 +49,11 @@ struct CmdDumpPath2 : Command return "serialise a path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To serialise directory 'foo' as a NAR:", - "nix nar dump-path ./foo" - }, - }; + return + #include "nar-dump-path.md" + ; } void run() override diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 1f5ed6913..df67240f9 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -75,6 +75,8 @@ struct MixLs : virtual Args, MixJSON if (json) { JSONPlaceholder jsonRoot(std::cout); + if (showDirectory) + throw UsageError("'--directory' is useless with '--json'"); listNar(jsonRoot, accessor, path, recursive); } else listText(accessor); @@ -127,14 +129,11 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } - Examples examples() override + std::string doc() override { - return { - Example{ - "To list a specific file in a NAR:", - "nix nar ls -l hello.nar /bin/hello" - }, - }; + return + #include "nar-ls.md" + ; } std::string description() override diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md new file mode 100644 index 000000000..55c481a28 --- /dev/null +++ b/src/nix/nar-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* List a file in a NAR and pipe it through `gunzip`: + + ```console + # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip + .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. + .TH HELLO "1" "November 2014" "hello 2.10" "User Commands" + … + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* inside the NAR file *nar*. + +)"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md new file mode 100644 index 000000000..26191ad25 --- /dev/null +++ b/src/nix/nar-dump-path.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* To serialise directory `foo` as a NAR: + + ```console + # nix nar dump-path ./foo > foo.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of +*path*, which must contain only regular files, directories and +symbolic links. The NAR is written to standard output. + +)"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md new file mode 100644 index 000000000..d373f9715 --- /dev/null +++ b/src/nix/nar-ls.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* To list a specific file in a NAR: + + ```console + # nix nar ls -l ./hello.nar /bin/hello + -r-xr-xr-x 38184 hello + ``` + +* To recursively list the contents of a directory inside a NAR, in JSON + format: + + ```console + # nix nar ls --json -R ./hello.nar /bin + {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} + ``` + +# Description + +This command shows information about a *path* inside NAR file *nar*. + +)"" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index e239ce96a..0775d3c25 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -9,7 +9,14 @@ struct CmdNar : NixMultiCommand std::string description() override { - return "query the contents of NAR files"; + return "create or inspect NAR files"; + } + + std::string doc() override + { + return + #include "nar.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/nar.md b/src/nix/nar.md new file mode 100644 index 000000000..a83b5c764 --- /dev/null +++ b/src/nix/nar.md @@ -0,0 +1,13 @@ +R""( + +# Description + +`nix nar` provides several subcommands for creating and inspecting +*Nix Archives* (NARs). + +# File format + +For the definition of the NAR file format, see Figure 5.2 in +https://edolstra.github.io/pubs/phd-thesis.pdf. + +)"" From c14ed3f8b2cbddb335227d2ff5188896e76b713f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 20:06:19 +0100 Subject: [PATCH 040/113] Add 'nix store' NAR-related manpages --- src/nix/cat.cc | 7 +++++++ src/nix/dump-path.cc | 11 ++++------- src/nix/ls.cc | 17 +++++++---------- src/nix/store-cat.md | 19 +++++++++++++++++++ src/nix/store-dump-path.md | 23 +++++++++++++++++++++++ src/nix/store-ls.md | 27 +++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 src/nix/store-cat.md create mode 100644 src/nix/store-dump-path.md create mode 100644 src/nix/store-ls.md diff --git a/src/nix/cat.cc b/src/nix/cat.cc index fe2f0a241..e28ee3c50 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -37,6 +37,13 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } + std::string doc() override + { + return + #include "store-cat.md" + ; + } + void run(ref store) override { cat(store->getFSAccessor()); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 63393ef9c..c4edc894b 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -11,14 +11,11 @@ struct CmdDumpPath : StorePathCommand return "serialise a store path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" - }, - }; + return + #include "store-dump-path.md" + ; } void run(ref store, const StorePath & storePath) override diff --git a/src/nix/ls.cc b/src/nix/ls.cc index df67240f9..d48287f27 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -94,21 +94,18 @@ struct CmdLsStore : StoreCommand, MixLs }); } - Examples examples() override - { - return { - Example{ - "To list the contents of a store path in a binary cache:", - "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" - }, - }; - } - std::string description() override { return "show information about a path in the Nix store"; } + std::string doc() override + { + return + #include "store-ls.md" + ; + } + void run(ref store) override { list(store->getFSAccessor()); diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md new file mode 100644 index 000000000..da2073473 --- /dev/null +++ b/src/nix/store-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Show the contents of a file in a binary cache: + + ```console + # nix store cat --store https://cache.nixos.org/ \ + /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1 + 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............| + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* in a Nix store. *path* can be a top-level store path or +any file inside a store path. + +)"" diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md new file mode 100644 index 000000000..4ef563526 --- /dev/null +++ b/src/nix/store-dump-path.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* To get a NAR containing the GNU Hello package: + + ```console + # nix store dump-path nixpkgs#hello > hello.nar + ``` + +* To get a NAR from the binary cache https://cache.nixos.org/: + + ```console + # nix store dump-path --store https://cache.nixos.org/ \ + /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of the +store path *installable*. The NAR is written to standard output. + +)"" diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md new file mode 100644 index 000000000..836efce42 --- /dev/null +++ b/src/nix/store-ls.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* To list the contents of a store path in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + dr-xr-xr-x 0 ./bin + -r-xr-xr-x 38184 ./bin/hello + dr-xr-xr-x 0 ./share + … + ``` + +* To show information about a specific file in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + -r-xr-xr-x 38184 hello + ``` + +# Description + +This command shows information about *path* in a Nix store. *path* can +be a top-level store path or any file inside a store path. + +)"" From 19540744ad164c13803048d5e8883fe68f9d2d10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 20:33:53 +0100 Subject: [PATCH 041/113] Add 'nix why-depends' manpage --- src/nix/why-depends.cc | 19 +++------- src/nix/why-depends.md | 80 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 src/nix/why-depends.md diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 57b9a2208..297b638cc 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -50,22 +50,11 @@ struct CmdWhyDepends : SourceExprCommand return "show why a package has another package in its closure"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs#hello nixpkgs#glibc" - }, - Example{ - "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11" - }, - Example{ - "To show why Glibc depends on itself:", - "nix why-depends nixpkgs#glibc nixpkgs#glibc" - }, - }; + return + #include "why-depends.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md new file mode 100644 index 000000000..dc13619e1 --- /dev/null +++ b/src/nix/why-depends.md @@ -0,0 +1,80 @@ +R""( + +# Examples + +* Show one path through the dependency graph leading from Hello to + Glibc: + + ```console + # nix why-depends nixpkgs#hello nixpkgs#glibc + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.… + → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32 + ``` + +* Show all files and paths in the dependency graph leading from + Thunderbird to libX11: + + ```console + # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11 + /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1 + ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf… + │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store… + │ → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5 + │ ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0… + │ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + … + ``` + +* Show why Glibc depends on itself: + + ```console + # nix why-depends nixpkgs#glibc nixpkgs#glibc + /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + └───lib/ld-2.31.so: …che Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache. --… + → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + ``` + +* Show why Geeqie has a build-time dependency on `systemd`: + + ```console + # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd + /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv + └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/… + → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv + └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix… + → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv + └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/… + → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv + ``` + +# Description + +Nix automatically determines potential runtime dependencies between +store paths by scanning for the *hash parts* of store paths. For +instance, if there exists a store path +`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file +inside another store path contains the string `9df65igw…`, then the +latter store path *refers* to the former, and thus might need it at +runtime. Nix always maintains the existence of the transitive closure +of a store path under the references relationship; it is therefore not +possible to install a store path without having all of its references +present. + +Sometimes Nix packages end up with unexpected runtime dependencies; +for instance, a reference to a compiler might accidentally end up in a +binary, causing the former to be in the latter's closure. This kind of +*closure size bloat* is undesirable. + +`nix why-depends` allows you to diagnose the cause of such issues. It +shows why the store path *package* depends on the store path +*dependency*, by showing a shortest sequence in the references graph +from the former to the latter. Also, for each node along this path, it +shows a file fragment containing a reference to the next store path in +the sequence. + +To show why derivation *package* has a build-time rather than runtime +dependency on derivation *dependency*, use `--derivation`. + +)"" From 6b32551aba5dfd6a912277297eb28cedc92da26d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 21:11:48 +0100 Subject: [PATCH 042/113] Add 'nix upgrade-nix' manpage --- src/nix/upgrade-nix.cc | 15 ++++----------- src/nix/upgrade-nix.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 src/nix/upgrade-nix.md diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 66ecc5b34..79be31e73 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand return "upgrade Nix to the latest stable version"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade Nix to the latest stable version:", - "nix upgrade-nix" - }, - Example{ - "To upgrade Nix in a specific profile:", - "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" - }, - }; + return + #include "upgrade-nix.md" + ; } Category category() override { return catNixInstallation; } diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md new file mode 100644 index 000000000..4d27daad9 --- /dev/null +++ b/src/nix/upgrade-nix.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Upgrade Nix to the latest stable version: + + ```console + # nix upgrade-nix + ``` + +* Upgrade Nix in a specific profile: + + ```console + # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + ``` + +# Description + +This command upgrades Nix to the latest version. By default, it +locates the directory containing the `nix` binary in the `$PATH` +environment variable. If that directory is a Nix profile, it will +upgrade the `nix` package in that profile to the latest stable binary +release. + +You cannot use this command to upgrade Nix in the system profile of a +NixOS system (that is, if `nix` is found in `/run/current-system`). + +)"" From 8dd7d7e9db8165c316b1ef168f57ed3632507fe2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 23:45:06 +0100 Subject: [PATCH 043/113] Add 'nix store verify' manpage --- src/nix/verify.cc | 15 ++++----------- src/nix/verify.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 src/nix/verify.md diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bcf85d7dd..16d42349f 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -35,18 +35,11 @@ struct CmdVerify : StorePathsCommand return "verify the integrity of store paths"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To verify the entire Nix store:", - "nix store verify --all" - }, - Example{ - "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix store verify -r -n2 --no-contents $(type -p firefox)" - }, - }; + return + #include "verify.md" + ; } void run(ref store, StorePaths storePaths) override diff --git a/src/nix/verify.md b/src/nix/verify.md new file mode 100644 index 000000000..1c43792e7 --- /dev/null +++ b/src/nix/verify.md @@ -0,0 +1,49 @@ +R""( + +# Examples + +* Verify the entire Nix store: + + ```console + # nix store verify --all + ``` + +* Check whether each path in the closure of Firefox has at least 2 + signatures: + + ```console + # nix store verify -r -n2 --no-contents $(type -p firefox) + ``` + +* Verify a store path in the binary cache `https://cache.nixos.org/`: + + ```console + # nix store verify --store https://cache.nixos.org/ \ + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +# Description + +This command verifies the integrity of the store paths *installables*, +or, if `--all` is given, the entire Nix store. For each path, it +checks that + +* its contents match the NAR hash recorded in the Nix database; and + +* it is *trusted*, that is, it is signed by at least one trusted + signing key, is content-addressed, or is built locally ("ultimately + trusted"). + +# Exit status + +The exit status of this command is the sum of the following values: + +* **1** if any path is corrupted (i.e. its contents don't match the + recorded NAR hash). + +* **2** if any path is untrusted. + +* **4** if any path couldn't be verified for any other reason (such as + an I/O error). + +)"" From cb25a89f1cb9b3a26d84b3429b309be2cfa513a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Dec 2020 23:54:11 +0100 Subject: [PATCH 044/113] Add 'nix store optimise' manpage --- src/nix/optimise-store.cc | 11 ++++------- src/nix/optimise-store.md | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 src/nix/optimise-store.md diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index bc7f175ac..985006e5a 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -13,14 +13,11 @@ struct CmdOptimiseStore : StoreCommand return "replace identical files in the store by hard links"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To optimise the Nix store:", - "nix store optimise" - }, - }; + return + #include "optimise-store.md" + ; } void run(ref store) override diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md new file mode 100644 index 000000000..f6fb66f97 --- /dev/null +++ b/src/nix/optimise-store.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Optimise the Nix store: + + ```console + nix store optimise + ``` + +# Description + +This command deduplicates the Nix store: it scans the store for +regular files with identical contents, and replaces them with hard +links to a single instance. + +Note that you can also set `auto-optimise-store` to `true` in +`nix.conf` to perform this optimisation incrementally whenever a new +path is added to the Nix store. To make this efficient, Nix maintains +a content-addressed index of all the files in the Nix store in the +directory `/nix/store/.links/`. + +)"" From 2e599dbb88855311c33c70460b82ec16487c9071 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 13:45:42 +0100 Subject: [PATCH 045/113] Add 'nix path-info' manpage --- src/nix/path-info.cc | 35 +++-------------- src/nix/path-info.md | 94 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 src/nix/path-info.md diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 63cf885f9..30b6a50f8 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -29,38 +29,15 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } - Category category() override { return catSecondary; } - - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", - "nix path-info -rS /run/current-system | sort -nk2" - }, - Example{ - "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs#rust" - }, - Example{ - "To check the existence of a path in a binary cache:", - "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" - }, - Example{ - "To print the 10 most recently added paths (using --json and the jq(1) command):", - "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" - }, - Example{ - "To show the size of the entire Nix store:", - "nix path-info --json --all | jq 'map(.narSize) | add'" - }, - Example{ - "To show every path whose closure is bigger than 1 GB, sorted by closure size:", - "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" - }, - }; + return + #include "path-info.md" + ; } + Category category() override { return catSecondary; } + void printSize(uint64_t value) { if (!humanReadable) { diff --git a/src/nix/path-info.md b/src/nix/path-info.md new file mode 100644 index 000000000..b4ba5862d --- /dev/null +++ b/src/nix/path-info.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Print the store path produced by `nixpkgs#hello`: + + ```console + # nix path-info nixpkgs#hello + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +* Show the closure sizes of every path in the current NixOS system + closure, sorted by size: + + ```console + # nix path-info -rS /run/current-system | sort -nk2 + /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 + /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 + … + /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc 5783255504 + /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65 5887562256 + ``` + +* Show a package's closure size and all its dependencies with human + readable sizes: + + ```console + # nix path-info -rsSh nixpkgs#rustc + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + … + ``` + +* Check the existence of a path in a binary cache: + + ```console + # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid + + ``` + +* Print the 10 most recently added paths (using --json and the jq(1) + command): + + ```console + # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + ``` + +* Show the size of the entire Nix store: + + ```console + # nix path-info --json --all | jq 'map(.narSize) | add' + 49812020936 + ``` + +* Show every path whose closure is bigger than 1 GB, sorted by closure + size: + + ```console + # nix path-info --json --all -S \ + | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + [ + …, + [ + "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + 5887562256 + ] + ] + ``` + +* Print the path of the store derivation produced by `nixpkgs#hello`: + + ```console + # nix path-info --derivation nixpkgs#hello + /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv + ``` + +# Description + +This command shows information about the store paths produced by +*installables*, or about all paths in the store if you pass `--all`. + +By default, this command only prints the store paths. You can get +additional information by passing flags such as `--closure-size`, +--size`, `--sigs` or `--json`. + +> **Warning** +> +> Note that `nix path-info` does not build or substitute the +> *installables* you specify. Thus, if the corresponding store paths +> don't already exist, this command will fail. You can use `nix build` +> to ensure that they exist. + +)"" From cdf20e04b7acc0efd7fa9640283103502ac80c53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 14:14:30 +0100 Subject: [PATCH 046/113] Doh --- src/nix/path-info.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/path-info.md b/src/nix/path-info.md index b4ba5862d..76a83e39d 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -73,7 +73,7 @@ R""( ```console # nix path-info --derivation nixpkgs#hello /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv - ``` + ``` # Description From e6bea9c9b10ded0e65981edf84cedd00ec86883a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 17:36:59 +0100 Subject: [PATCH 047/113] Add 'nix store make-content-addressable' manpage --- src/nix/make-content-addressable.cc | 17 +++------ src/nix/make-content-addressable.md | 59 +++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 src/nix/make-content-addressable.md diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5165c4804..f5bdc7e65 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -15,21 +15,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON std::string description() override { - return "rewrite a path or closure to content-addressable form"; + return "rewrite a path or closure to content-addressed form"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix store make-content-addressable nixpkgs#hello" - }, - Example{ - "To compute a content-addressable representation of the current NixOS system closure:", - "nix store make-content-addressable -r /run/current-system" - }, - }; + return + #include "make-content-addressable.md" + ; } void run(ref store, StorePaths storePaths) override diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md new file mode 100644 index 000000000..3dd847edc --- /dev/null +++ b/src/nix/make-content-addressable.md @@ -0,0 +1,59 @@ +R""( + +# Examples + +* Create a content-addressed representation of the closure of GNU Hello: + + ```console + # nix store make-content-addressable -r nixpkgs#hello + … + rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' + ``` + + Since the resulting paths are content-addressed, they are always + trusted and don't need signatures to copied to another store: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 + ``` + + By contrast, the original closure is input-addressed, so it does + need signatures to be trusted: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + ``` + +* Create a content-addressed representation of the current NixOS + system closure: + + ```console + # nix store make-content-addressable -r /run/current-system + ``` + +# Description + +This command converts the closure of the store paths specified by +*installables* to content-addressed form. Nix store paths are usually +*input-addressed*, meaning that the hash part of the store path is +computed from the contents of the derivation (i.e., the build-time +dependency graph). Input-addressed paths need to be signed by a +trusted key if you want to import them into a store, because we need +to trust that the contents of the path were actually built by the +derivation. + +By contrast, in a *content-addressed* path, the hash part is computed +from the contents of the path. This allows the contents of the path to +be verified without any additional information such as +signatures. This means that a command like + +```console +# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ + --substituters https://my-cache.example.org +``` + +will succeed even if the binary cache `https://my-cache.example.org` +doesn't present any signatures. + +)"" From daf365b0b731bb3ac86128a965394dcff8d6f5b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 18:40:16 +0100 Subject: [PATCH 048/113] Add 'nix help' manpage --- src/nix/help.md | 17 +++++++++++++++++ src/nix/main.cc | 17 +++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 src/nix/help.md diff --git a/src/nix/help.md b/src/nix/help.md new file mode 100644 index 000000000..734f35028 --- /dev/null +++ b/src/nix/help.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* Show help about `nix` in general: + + ```console + # nix help + ``` + +* Show help about a particular subcommand: + + ```console + # nix help flake info + ``` + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index e7a15dec9..afe7cb8d7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -205,21 +205,14 @@ struct CmdHelp : Command std::string description() override { - return "show help about 'nix' or a particular subcommand"; + return "show help about `nix` or a particular subcommand"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show help about 'nix' in general:", - "nix help" - }, - Example{ - "To show help about a particular subcommand:", - "nix help run" - }, - }; + return + #include "help.md" + ; } void run() override From 3b123a6ee63aef29d61a3dac7f2f01c2e92cd6d0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:13:23 +0100 Subject: [PATCH 049/113] nix show-derivation: Say "system" instead of "platform" There is really no good reason to use "platform" except that that's what we use internally (also for no good reason). --- src/nix/show-derivation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 8e1a58ac2..211e6a27a 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -103,7 +103,7 @@ struct CmdShowDerivation : InstallablesCommand } } - drvObj.attr("platform", drv.platform); + drvObj.attr("system", drv.platform); drvObj.attr("builder", drv.builder); { From 4f3e7f4eec9ef5fb86aea9f745a3574cc5cfae28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:14:18 +0100 Subject: [PATCH 050/113] Add 'nix show-derivation' manpage --- src/nix/show-derivation.cc | 15 ++---- src/nix/show-derivation.md | 103 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 src/nix/show-derivation.md diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 211e6a27a..13f2c8e69 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -29,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand return "show the contents of a store derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs#hello" - }, - Example{ - "To show the full derivation graph (if available) that produced your NixOS system:", - "nix show-derivation -r /run/current-system" - }, - }; + return + #include "show-derivation.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md new file mode 100644 index 000000000..aa863899c --- /dev/null +++ b/src/nix/show-derivation.md @@ -0,0 +1,103 @@ +R""( + +# Examples + +* Show the store derivation that results from evaluating the Hello + package: + + ```console + # nix show-derivation nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix show-derivation -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix show-derivation -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +store derivations to which *installables* evaluate. Store derivations +are used internally by Nix. They are store paths with extension `.drv` +that represent the build-time dependency graph to which a Nix +expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" From f4e9d4fcb3e393af2736f28fc41e4e3b79a8e60d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Dec 2020 19:58:04 +0100 Subject: [PATCH 051/113] Add 'nix store diff-closures' manpage --- src/nix/diff-closures.cc | 11 ++++----- src/nix/diff-closures.md | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/nix/diff-closures.md diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index f72b5eff7..0c7d531c1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -121,14 +121,11 @@ struct CmdDiffClosures : SourceExprCommand return "show what packages and versions were added and removed between two closures"; } - Examples examples() override + std::string doc() override { - return { - { - "To show what got added and removed between two versions of the NixOS system profile:", - "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", - }, - }; + return + #include "diff-closures.md" + ; } void run(ref store) override diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md new file mode 100644 index 000000000..0294c0d8d --- /dev/null +++ b/src/nix/diff-closures.md @@ -0,0 +1,51 @@ +R""( + +# Examples + +* Show what got added and removed between two versions of the NixOS + system profile: + + ```console + # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link + acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18 + baloo-widgets: 20.08.1 → 20.08.2 + bluez-qt: +12.6 KiB + dolphin: 20.08.1 → 20.08.2, +13.9 KiB + kdeconnect: 20.08.2 → ∅, -6597.8 KiB + kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + … + ``` + +# Description + +This command shows the differences between the two closures *before* +and *after* with respect to the addition, removal, or version change +of packages, as well as changes in store path sizes. + +For each package name in the two closures (where a package name is +defined as the name component of a store path excluding the version), +if there is a change in the set of versions of the package, or a +change in the size of the store paths of more than 8 KiB, it prints a +line like this: + +```console +dolphin: 20.08.1 → 20.08.2, +13.9 KiB +``` + +No size change is shown if it's below the threshold. If the package +does not exist in either the *before* or *after* closures, it is +represented using `∅` (empty set) on the appropriate side of the +arrow. If a package has an empty version string, the version is +rendered as `ε` (epsilon). + +There may be multiple versions of a package in each closure. In that +case, only the changed versions are shown. Thus, + +```console +libfoo: 1.2, 1.3 → 1.4 +``` + +leaves open the possibility that there are other versions (e.g. `1.1`) +that exist in both closures. + +)"" From 0c09f63de84e15c15e8621a53d6d25f023b4ad06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Dec 2020 11:45:59 +0100 Subject: [PATCH 052/113] Add 'nix bundle' manpage Fixes #4375. --- src/nix/bundle.cc | 11 ++++------- src/nix/bundle.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/nix/bundle.md diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index eddd82f40..5f558b01e 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -40,14 +40,11 @@ struct CmdBundle : InstallableCommand return "bundle an application so that it works outside of the Nix store"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To bundle Hello:", - "nix bundle hello" - }, - }; + return + #include "bundle.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/bundle.md b/src/nix/bundle.md new file mode 100644 index 000000000..c183a170d --- /dev/null +++ b/src/nix/bundle.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Bundle Hello: + + ```console + # nix bundle nixpkgs#hello + # ./hello + Hello, world! + ``` + +* Bundle a specific version of Nix: + + ```console + # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec + # ./nix --version + nix (Nix) 2.4pre20201215_e3ddffb + ``` + +# Description + +`nix bundle` packs the closure of the [Nix app](./nix3-run.md) +*installable* into a single self-extracting executable. See the +[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle) +for more details. + +> **Note** +> +> This command only works on Linux. + +)"" From 16e34085e8f45758b97c41cfcd720552c68a3c98 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Dec 2020 14:25:36 +0100 Subject: [PATCH 053/113] Add 'nix profile' manpage --- src/nix/profile-diff-closures.md | 28 ++++++++ src/nix/profile-info.md | 31 +++++++++ src/nix/profile-install.md | 27 ++++++++ src/nix/profile-remove.md | 32 +++++++++ src/nix/profile-upgrade.md | 41 ++++++++++++ src/nix/profile.cc | 90 +++++++++----------------- src/nix/profile.md | 107 +++++++++++++++++++++++++++++++ 7 files changed, 295 insertions(+), 61 deletions(-) create mode 100644 src/nix/profile-diff-closures.md create mode 100644 src/nix/profile-info.md create mode 100644 src/nix/profile-install.md create mode 100644 src/nix/profile-remove.md create mode 100644 src/nix/profile-upgrade.md create mode 100644 src/nix/profile.md diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md new file mode 100644 index 000000000..295d1252b --- /dev/null +++ b/src/nix/profile-diff-closures.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Show what changed between each version of the NixOS system + profile: + + ```console + # nix profile diff-closures --profile /nix/var/nix/profiles/system + Version 13 -> 14: + acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14 + aws-sdk-cpp: -6723.1 KiB + … + + Version 14 -> 15: + acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16 + attica: -996.2 KiB + breeze-icons: -78713.5 KiB + brotli: 1.0.7 → 1.0.9, +44.2 KiB + ``` + +# Description + +This command shows the difference between the closures of subsequent +versions of a profile. See [`nix store +diff-closures`](nix3-store-diff-closures.md) for details. + +)"" diff --git a/src/nix/profile-info.md b/src/nix/profile-info.md new file mode 100644 index 000000000..a0c04fc8c --- /dev/null +++ b/src/nix/profile-info.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Show what packages are installed in the default profile: + + ```console + # nix profile info + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 + 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 + 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + ``` + +# Description + +This command shows what packages are currently installed in a +profile. The output consists of one line per package, with the +following fields: + +* An integer that can be used to unambiguously identify the package in + invocations of `nix profile remove` and `nix profile upgrade`. + +* The original ("mutable") flake reference and output attribute path + used at installation time. + +* The immutable flake reference to which the mutable flake reference + was resolved. + +* The store path(s) of the package. + +)"" diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md new file mode 100644 index 000000000..e3009491e --- /dev/null +++ b/src/nix/profile-install.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* Install a package from Nixpkgs: + + ```console + # nix profile install nixpkgs#hello + ``` + +* Install a package from a specific branch of Nixpkgs: + + ```console + # nix profile install nixpkgs/release-20.09#hello + ``` + +* Install a package from a specific revision of Nixpkgs: + + ```console + # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello + ``` + +# Description + +This command adds *installables* to a Nix profile. + +)"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md new file mode 100644 index 000000000..dcf825da9 --- /dev/null +++ b/src/nix/profile-remove.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Remove a package by position: + + ```console + # nix profile remove 3 + ``` + +* Remove a package by attribute path: + + ```console + # nix profile remove packages.x86_64-linux.hello + ``` + +* Remove all packages: + ```console + # nix profile remove '.*' + ``` + +* Remove a package by store path: + + ```console + # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10 + ``` + +# Description + +This command removes a package from a profile. + +)"" diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md new file mode 100644 index 000000000..2bd5d256d --- /dev/null +++ b/src/nix/profile-upgrade.md @@ -0,0 +1,41 @@ +R""( + +# Examples + +* Upgrade all packages that were installed using a mutable flake + reference: + + ```console + # nix profile upgrade '.*' + ``` + +* Upgrade a specific package: + + ```console + # nix profile upgrade packages.x86_64-linux.hello + ``` + +* Upgrade a specific profile element by number: + + ```console + # nix profile info + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … + + # nix profile upgrade 0 + ``` + +# Description + +This command upgrades a previously installed package in a Nix profile, +by fetching and evaluating the latest version of the flake from which +the package was installed. + +> **Warning** +> +> This only works if you used a *mutable* flake reference at +> installation time, e.g. `nixpkgs#hello`. It does not work if you +> used an *immutable* flake reference +> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), +> since in that case the "latest version" is always the same. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cf5ccd62..d8d2b3a70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -151,22 +151,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile return "install a package into a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To install a package from Nixpkgs:", - "nix profile install nixpkgs#hello" - }, - Example{ - "To install a package from a specific branch of Nixpkgs:", - "nix profile install nixpkgs/release-19.09#hello" - }, - Example{ - "To install a package from a specific revision of Nixpkgs:", - "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" - }, - }; + return + #include "profile-install.md" + ; } void run(ref store) override @@ -257,26 +246,11 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem return "remove packages from a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To remove a package by attribute path:", - "nix profile remove packages.x86_64-linux.hello" - }, - Example{ - "To remove all packages:", - "nix profile remove '.*'" - }, - Example{ - "To remove a package by store path:", - "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" - }, - Example{ - "To remove a package by position:", - "nix profile remove 3" - }, - }; + return + #include "profile-remove.md" + ; } void run(ref store) override @@ -310,18 +284,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf return "upgrade packages using their most recent flake"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade all packages that were installed using a mutable flake reference:", - "nix profile upgrade '.*'" - }, - Example{ - "To upgrade a specific package:", - "nix profile upgrade packages.x86_64-linux.hello" - }, - }; + return + #include "profile-upgrade.md" + ; } void run(ref store) override @@ -377,14 +344,11 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro return "list installed packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what packages are installed in the default profile:", - "nix profile info" - }, - }; + return + #include "profile-info.md" + ; } void run(ref store) override @@ -405,17 +369,14 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile { std::string description() override { - return "show the closure difference between each generation of a profile"; + return "show the closure difference between each version of a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what changed between each generation of the NixOS system profile:", - "nix profile diff-closures --profile /nix/var/nix/profiles/system" - }, - }; + return + #include "profile-diff-closures.md" + ; } void run(ref store) override @@ -429,7 +390,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile if (prevGen) { if (!first) std::cout << "\n"; first = false; - std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number); + std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -458,6 +419,13 @@ struct CmdProfile : NixMultiCommand return "manage Nix profiles"; } + std::string doc() override + { + return + #include "profile.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/profile.md b/src/nix/profile.md new file mode 100644 index 000000000..d3ddcd3d1 --- /dev/null +++ b/src/nix/profile.md @@ -0,0 +1,107 @@ +R""( + +# Description + +`nix profile` allows you to create and manage *Nix profiles*. A Nix +profile is a set of packages that can be installed and upgraded +independently from each other. Nix profiles are versioned, allowing +them to be rolled back easily. + +# Default profile + +The default profile used by `nix profile` is `$HOME/.nix-profile`, +which, if it does not exist, is created as a symlink to +`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. + +You can specify another profile location using `--profile` *path*. + +# Filesystem layout + +Profiles are versioned as follows. When using profile *path*, *path* +is a symlink to *path*`-`*N*, where *N* is the current *version* of +the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix +store. For example: + +```console +$ ls -l /nix/var/nix/profiles/per-user/alice/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ +/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +The file `manifest.json` records the provenance of the packages that +are installed in this version of the profile. It looks like this: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUri": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUri`: The [flake reference](./nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The immutable flake reference to which `originalUri` + resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `defaultPackage.x86_64-linux`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. + +)"" From 629af83b2d86d77305dc994b83f176a377106c3e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 14 Jul 2020 20:59:24 +0200 Subject: [PATCH 054/113] Provide a more meaningful error-message for `builtins.fetchGit` if a revision can't be checked out A common pitfall when using e.g. `builtins.fetchGit` is the `fatal: not a tree object`-error when trying to fetch a revision of a git-repository that isn't on the `master` branch and no `ref` is specified. In order to make clear what's the problem, I added a simple check whether the revision in question exists and if it doesn't a more meaningful error-message is displayed: ``` nix-repl> builtins.fetchGit { url = "https://github.com/owner/myrepo"; rev = ""; } moderror: --- Error -------------------------------------------------------------------- nix Cannot find Git revision 'bf1cc5c648e6aed7360448a3745bb2fe4fbbf0e9' in ref 'master' of repository 'https://gitlab.com/Ma27/nvim.nix'! Please make sure that the rev exists on the ref you've specified or add allRefs = true; to fetchGit. ``` Closes #2431 --- src/libfetchers/git.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e7712c5fd..1f298c2d6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -392,6 +392,28 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + RunOptions checkCommitOpts( + "git", + { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() } + ); + checkCommitOpts.searchPath = true; + checkCommitOpts.mergeStderrToStdout = true; + + auto result = runProgram(checkCommitOpts); + if (WEXITSTATUS(result.first) == 128 + && result.second.find("bad file") != std::string::npos + ) { + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + input.getRev()->gitRev(), + *input.getRef(), + actualUrl + ); + } + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); From 2857b1baaf78bbadeec01adfae8a50fc0f2a254f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 17 Jul 2020 20:34:57 +0200 Subject: [PATCH 055/113] Add explicit `allRefs = true;` argument to `fetchGit` Sometimes it's necessary to fetch a git repository at a revision and it's unknown which ref contains the revision in question. An example would be a Cargo.lock which only provides the URL and the revision when using a git repository as build input. However it's considered a bad practice to perform a full checkout of a repository since this may take a lot of time and can eat up a lot of disk space. This patch makes a full checkout explicit by adding an `allRefs` argument to `builtins.fetchGit` which fetches all refs if explicitly set to true. Closes #2409 --- src/libfetchers/git.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f298c2d6..81c647f89 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); + maybeGetBoolAttr(attrs, "allRefs"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -169,10 +170,12 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; if (submodules) cacheType += "-submodules"; + if (allRefs) cacheType += "-all-refs"; auto getImmutableAttrs = [&]() { @@ -338,11 +341,15 @@ struct GitInputScheme : InputScheme } } } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + if (allRefs) { + doFetch = true; + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } } if (doFetch) { @@ -352,9 +359,11 @@ struct GitInputScheme : InputScheme // we're using --quiet for now. Should process its stderr. try { auto ref = input.getRef(); - auto fetchRef = ref->compare(0, 5, "refs/") == 0 - ? *ref - : "refs/heads/" + *ref; + auto fetchRef = allRefs + ? "refs/*" + : ref->compare(0, 5, "refs/") == 0 + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; From 724b7f4fb660212a97ba6482208c299158720c5b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 11:15:29 +0100 Subject: [PATCH 056/113] Don't log from inside the logger This deadlocks ProgressBar, e.g. # nix run --impure --no-substitute --store '/tmp/nix2?store=/foo' --expr 'derivation { builder = /nix/store/zi90rxslsm4mlr46l2xws1rm94g7pk8p-busybox-1.31.1-x86_64-unknown-linux-musl/bin/busybox; }' leads to Thread 1 (Thread 0x7ffff6126e80 (LWP 12250)): #0 0x00007ffff7215d62 in __lll_lock_wait () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0 #1 0x00007ffff720e721 in pthread_mutex_lock () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0 #2 0x00007ffff7ad17fa in __gthread_mutex_lock (__mutex=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/x86_64-unknown-linux-gnu/bits/gthr-default.h:749 #3 std::mutex::lock (this=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/std_mutex.h:100 #4 std::unique_lock::lock (this=0x7fffffff09a8, this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:141 #5 std::unique_lock::unique_lock (__m=..., this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:71 #6 nix::Sync::Lock::Lock (s=0x6c5448, this=0x7fffffff09a0) at src/libutil/sync.hh:45 #7 nix::Sync::lock (this=0x6c5448) at src/libutil/sync.hh:85 #8 nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:131 #9 0x00007ffff7608cfd in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88 #10 nix::getCodeLines (errPos=...) at src/libutil/error.cc:66 #11 0x00007ffff76073f2 in nix::showErrorInfo (out=..., einfo=..., showTrace=) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/optional:897 #12 0x00007ffff7ad19e7 in nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:134 #13 0x00007ffff7ab9d10 in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88 #14 nix::handleExceptions(std::__cxx11::basic_string, std::allocator > const&, std::function) (programName="/home/eelco/Dev/nix/outputs/out/bin/nix", fun=...) at src/libmain/shared.cc:328 #15 0x000000000046226b in main (argc=, argv=) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/ext/new_allocator.h:80 --- src/libutil/error.cc | 50 +++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 803a72953..e7dc3f1d3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -62,35 +62,28 @@ std::optional getCodeLines(const ErrPos & errPos) LinesOfCode loc; try { AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) { - logError(SysError("opening file '%1%'", errPos.file).info()); - return std::nullopt; - } - else + if (!fd) return {}; + + // count the newlines. + int count = 0; + string line; + int pl = errPos.line - 1; + do { - // count the newlines. - int count = 0; - string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - { - ; - } - else if (count == pl) { - loc.prevLineOfCode = line; - } else if (count == pl + 1) { - loc.errLineOfCode = line; - } else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } + line = readLine(fd.get()); + ++count; + if (count < pl) + ; + else if (count == pl) + loc.prevLineOfCode = line; + else if (count == pl + 1) + loc.errLineOfCode = line; + else if (count == pl + 2) { + loc.nextLineOfCode = line; + break; + } + } while (true); + return loc; } catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) @@ -99,7 +92,6 @@ std::optional getCodeLines(const ErrPos & errPos) return std::nullopt; } catch (std::exception & e) { - printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } } else { From e54971d019821dd213db028a80cd5fca85ee2ed6 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 8 Sep 2020 19:45:28 +0200 Subject: [PATCH 057/113] Document `allRefs` argument of `builtins.fetchTree` --- src/libexpr/primops/fetchTree.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6e7ddde8e..133299030 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -324,6 +324,11 @@ static RegisterPrimOp primop_fetchGit({ A Boolean parameter that specifies whether submodules should be checked out. Defaults to `false`. + - allRefs + Whether to fetch all refs of the repository. With this argument being + true, it's possible to load a `rev` from *any* `ref` (by default only + `rev`s from the specified `ref` are supported). + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: From 897ae235fc2cef0ce711470a7b620241d82a1b09 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 22 Dec 2020 12:18:10 +0100 Subject: [PATCH 058/113] tests/fetchGit: test behavior of `allRefs = true;` --- tests/fetchGit.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 76390fa59..1e8963d76 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -41,6 +41,19 @@ export _NIX_FORCE_HTTP=1 path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] +# Fetch a rev from another branch +git -C $repo checkout -b devtest +echo "different file" >> $TEST_ROOT/git/differentbranch +git -C $repo add differentbranch +git -C $repo commit -m 'Test2' +git -C $repo checkout master +devrev=$(git -C $repo rev-parse devtest) +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'Cannot find Git revision' ]] + +[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] + # In pure eval mode, fetchGit without a revision should fail. [[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") = world ]] (! nix eval --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") From 5373f4be3b1427ed73303448a1fa801726f4dfa0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 12:28:50 +0100 Subject: [PATCH 059/113] chrootHelper: Handle symlinks in the root directory This is necessary on Ubuntu where /bin and /lib* are symlinks. --- src/nix/run.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nix/run.cc b/src/nix/run.cc index 92a52c6cd..ec61fc79a 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -258,14 +258,16 @@ void chrootHelper(int argc, char * * argv) for (auto entry : readDirectory("/")) { auto src = "/" + entry.name; - auto st = lstat(src); - if (!S_ISDIR(st.st_mode)) continue; Path dst = tmpDir + "/" + entry.name; if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); - if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); + auto st = lstat(src); + if (S_ISDIR(st.st_mode)) { + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } else if (S_ISLNK(st.st_mode)) + createSymlink(readLink(src), dst); } char * cwd = getcwd(0, 0); From 75efa421340b8fb2be6cf5351d3ef36a93b294e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 14:43:20 +0100 Subject: [PATCH 060/113] Move into the nix binary This makes the statically linked nix binary just work, without needing any additional files. --- Makefile | 1 - corepkgs/local.mk | 4 ---- src/libexpr/eval.cc | 7 ++----- src/libexpr/eval.hh | 2 ++ {corepkgs => src/libexpr}/fetchurl.nix | 0 src/libexpr/local.mk | 2 +- src/libexpr/parser.y | 4 ++++ src/libexpr/primops.cc | 10 +++++++++- tests/fetchurl.sh | 14 +++++++------- tests/lang/eval-okay-search-path.nix | 5 ++--- 10 files changed, 27 insertions(+), 22 deletions(-) delete mode 100644 corepkgs/local.mk rename {corepkgs => src/libexpr}/fetchurl.nix (100%) diff --git a/Makefile b/Makefile index c50d2c40f..f80b8bb82 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ makefiles = \ src/nix/local.mk \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ - corepkgs/local.mk \ misc/bash/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ diff --git a/corepkgs/local.mk b/corepkgs/local.mk deleted file mode 100644 index 0bc91cfab..000000000 --- a/corepkgs/local.mk +++ /dev/null @@ -1,4 +0,0 @@ -corepkgs_FILES = \ - fetchurl.nix - -$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5a641d02c..ead5bf8c7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -402,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref store) for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - try { - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - } catch (Error &) { - } - if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); @@ -457,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_) */ Path abspath = canonPath(path_); + if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e1f61baa..e3eaed6d3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -432,4 +432,6 @@ struct EvalSettings : Config extern EvalSettings evalSettings; +static const std::string corepkgsPrefix{"/__corepkgs__/"}; + } diff --git a/corepkgs/fetchurl.nix b/src/libexpr/fetchurl.nix similarity index 100% rename from corepkgs/fetchurl.nix rename to src/libexpr/fetchurl.nix diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 519da33f7..26c53d301 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -40,6 +40,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index a4c84c526..85eb05d61 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -698,6 +698,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } + + if (hasPrefix(path, "nix/")) + return corepkgsPrefix + path.substr(4); + throw ThrownError({ .hint = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 18438f681..c73a94f4e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -164,7 +164,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS state.forceFunction(**fun, pos); mkApp(v, **fun, w); state.forceAttrs(v, pos); - } else { + } + + else if (path == corepkgsPrefix + "fetchurl.nix") { + state.eval(state.parseExprFromString( + #include "fetchurl.nix.gen.hh" + , "/"), v); + } + + else { if (!vScope) state.evalFile(realPath, v); else { diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 10ec0173a..cd84e9a4c 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,7 +5,7 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -14,7 +14,7 @@ clearStore hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -25,7 +25,7 @@ hash=$(nix hash file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) +outPath=$(nix-build --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) cmp $outPath fetchurl.sh @@ -38,10 +38,10 @@ hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store store add-file ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ +nix-build --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ --no-out-link --substituters $other_store # Test unpacking a NAR. @@ -55,7 +55,7 @@ nix-store --dump $TEST_ROOT/archive > $nar hash=$(nix-hash --flat --type sha256 $nar) -outPath=$(nix-build '' --argstr url file://$nar --argstr sha256 $hash \ +outPath=$(nix-build --expr 'import ' --argstr url file://$nar --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) echo $outPath | grep -q 'xyzzy' @@ -69,7 +69,7 @@ nix-store --delete $outPath narxz=$TEST_ROOT/archive.nar.xz rm -f $narxz xz --keep $nar -outPath=$(nix-build '' --argstr url file://$narxz --argstr sha256 $hash \ +outPath=$(nix-build --expr 'import ' --argstr url file://$narxz --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) test -x $outPath/fetchurl.sh diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index c5a123d04..6fe33decc 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -1,10 +1,9 @@ with import ./lib.nix; with builtins; -assert pathExists ; +assert isFunction (import ); -assert length __nixPath == 6; -assert length (filter (x: x.prefix == "nix") __nixPath) == 1; +assert length __nixPath == 5; assert length (filter (x: baseNameOf x.path == "dir4") __nixPath) == 1; import + import + import + import From e27044216bf597710893e0366dbff60efbdaf0a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Dec 2020 16:23:57 +0100 Subject: [PATCH 061/113] Fix tests --- src/libutil/tests/logging.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..5b32c84a4 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -49,7 +49,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); + ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); } TEST(logEI, appendingHintsToPreviousError) { @@ -208,7 +208,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } TEST(logError, logErrorWithOnlyHintAndName) { @@ -290,7 +290,7 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); } TEST(addTrace, hideTracesWithoutShowTrace) { From c9279b831e91a762851464826eaa8a8e30979578 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 13:19:53 +0100 Subject: [PATCH 062/113] Add 'nix flake' manpages --- src/nix/flake-archive.md | 29 ++ src/nix/flake-check.md | 68 +++++ src/nix/flake-clone.md | 18 ++ src/nix/flake-info.md | 99 ++++++ src/nix/flake-init.md | 54 ++++ src/nix/flake-list-inputs.md | 23 ++ src/nix/flake-new.md | 34 +++ src/nix/flake-show.md | 38 +++ src/nix/flake-update.md | 53 ++++ src/nix/flake.cc | 94 ++++-- src/nix/flake.md | 566 +++++++++++++++++++++++++++++++++++ 11 files changed, 1046 insertions(+), 30 deletions(-) create mode 100644 src/nix/flake-archive.md create mode 100644 src/nix/flake-check.md create mode 100644 src/nix/flake-clone.md create mode 100644 src/nix/flake-info.md create mode 100644 src/nix/flake-init.md create mode 100644 src/nix/flake-list-inputs.md create mode 100644 src/nix/flake-new.md create mode 100644 src/nix/flake-show.md create mode 100644 src/nix/flake-update.md create mode 100644 src/nix/flake.md diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md new file mode 100644 index 000000000..85bbeeb16 --- /dev/null +++ b/src/nix/flake-archive.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Copy the `dwarffs` flake and its dependencies to a binary cache: + + ```console + # nix flake archive --to file:///tmp/my-cache dwarffs + ``` + +* Fetch the `dwarffs` flake and its dependencies to the local Nix + store: + + ```console + # nix flake archive dwarffs + ``` + +* Print the store paths of the flake sources of NixOps without + fetching them: + + ```console + # nix flake archive --json --dry-run nixops + ``` + +# Description + +FIXME + +)"" diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md new file mode 100644 index 000000000..dc079ba0c --- /dev/null +++ b/src/nix/flake-check.md @@ -0,0 +1,68 @@ +R""( + +# Examples + +* Evaluate the flake in the current directory, and build its checks: + + ```console + # nix flake check + ``` + +* Verify that the `patchelf` flake evaluates, but don't build its + checks: + + ```console + # nix flake check --no-build github:NixOS/patchelf + ``` + +# Description + +This command verifies that the flake specified by flake reference +*flake-url* can be evaluated successfully (as detailed below), and +that the derivations specified by the flake's `checks` output can be +built successfully. + +# Evaluation checks + +This following flake output attributes must be derivations: + +* `checks.`*system*`.`*name* +* `defaultPackage.`*system*` +* `devShell.`*system*` +* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `packages.`*system*`.`*name* + +The following flake output attributes must be [app +definitions](./nix3-run.md): + +* `apps.`*system*`.`*name* +* `defaultApp.`*system*` + +The following flake output attributes must be [template +definitions](./nix3-flake-init.md): + +* `defaultTemplate` +* `templates`.`*name* + +The following flake output attributes must be *Nixpkgs overlays*: + +* `overlay` +* `overlays`.`*name* + +The following flake output attributes must be *NixOS modules*: + +* `nixosModule` +* `nixosModules`.`*name* + +The following flake output attributes must be +[bundlers](./nix3-bundle.md): + +* `bundlers`.`*name* +* `defaultBundler` + +In addition, the `hydraJobs` output is evaluated in the same way as +Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested +attribute set of derivations). Similarly, the +`legacyPackages`.*system* output is evaluated like `nix-env -qa`. + +)"" diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md new file mode 100644 index 000000000..36cb96051 --- /dev/null +++ b/src/nix/flake-clone.md @@ -0,0 +1,18 @@ +R""( + +# Examples + +* Check out the source code of the `dwarffs` flake and build it: + + ```console + # nix flake clone dwarffs --dest dwarffs + # cd dwarffs + # nix build + ``` + +# Description + +This command performs a Git or Mercurial clone of the repository +containing the source code of the flake *flake-url*. + +)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-info.md new file mode 100644 index 000000000..fda3171db --- /dev/null +++ b/src/nix/flake-info.md @@ -0,0 +1,99 @@ +R""( + +# Examples + +* Show what `nixpkgs` resolves to: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs + Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Description: A collection of packages for the Nix package manager + Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source + Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Last modified: 2020-12-23 12:36:12 + ``` + +* Show information about `dwarffs` in JSON format: + + ```console + # nix flake info dwarffs --json | jq . + { + "description": "A filesystem that fetches DWARF debug info from the Internet on demand", + "lastModified": 1597153508, + "locked": { + "lastModified": 1597153508, + "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=", + "owner": "edolstra", + "repo": "dwarffs", + "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "type": "github" + }, + "original": { + "id": "dwarffs", + "type": "indirect" + }, + "originalUrl": "flake:dwarffs", + "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source", + "resolved": { + "owner": "edolstra", + "repo": "dwarffs", + "type": "github" + }, + "resolvedUrl": "github:edolstra/dwarffs", + "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5" + } + ``` + +# Description + +This command shows information about the flake specified by the flake +reference *flake-url*. It resolves the flake reference using the +[flake registry](./nix3-registry.md), fetches it, and prints some meta +data. This includes: + +* `Resolved URL`: If *flake-url* is a flake identifier, then this is + the flake reference that specifies its actual location, looked up in + the flake registry. + +* `Locked URL`: A flake reference that contains a commit or content + hash and thus uniquely identifies a specific flake version. + +* `Description`: A one-line description of the flake, taken from the + `description` field in `flake.nix`. + +* `Path`: The store path containing the source code of the flake. + +* `Revision`: The Git or Mercurial commit hash of the locked flake. + +* `Revisions`: The number of ancestors of the Git or Mercurial commit + of the locked flake. Note that this is not available for `github` + flakes. + +* `Last modified`: For Git or Mercurial flakes, this is the commit + time of the commit of the locked flake; for tarball flakes, it's the + most recent timestamp of any file inside the tarball. + +With `--json`, the output is a JSON object with the following fields: + +* `original` and `originalUrl`: The flake reference specified by the + user (*flake-url*) in attribute set and URL representation. + +* `resolved` and `resolvedUrl`: The resolved flake reference (see + above) in attribute set and URL representation. + +* `locked` and `lockedUrl`: The locked flake reference (see above) in + attribute set and URL representation. + +* `description`: See `Description` above. + +* `path`: See `Path` above. + +* `revision`: See `Revision` above. + +* `revCount`: See `Revisions` above. + +* `lastModified`: See `Last modified` above. + +)"" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md new file mode 100644 index 000000000..c66154ad5 --- /dev/null +++ b/src/nix/flake-init.md @@ -0,0 +1,54 @@ +R""( + +# Examples + +* Create a flake using the default template: + + ```console + # nix flake init + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template: + + ```console + # nix flake init -t templates#simpleContainer + ``` + +# Description + +This command creates a flake in the current directory by copying the +files of a template. It will not overwrite existing files. The default +template is `templates#defaultTemplate`, but this can be overriden +using `-t`. + +# Template definitions + +A flake can declare templates through its `templates` and +`defaultTemplate` output attributes. A template has two attributes: + +* `description`: A one-line description of the template, in CommonMark + syntax. + +* `path`: The path of the directory to be copied. + +Here is an example: + +``` +outputs = { self }: { + + templates.rust = { + path = ./rust; + description = "A simple Rust/Cargo project"; + }; + + templates.defaultTemplate = self.templates.rust; +} +``` + +)"" diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md new file mode 100644 index 000000000..250e13be0 --- /dev/null +++ b/src/nix/flake-list-inputs.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Show the inputs of the `hydra` flake: + + ```console + # nix flake list-inputs github:NixOS/hydra + github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 + ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' + ``` + +# Description + +This command shows the inputs of the flake specified by the flake +referenced *flake-url*. Since it prints the locked inputs that result +from generating or updating the lock file, this command essentially +displays the contents of the flake's lock file in human-readable form. + +)"" diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md new file mode 100644 index 000000000..725695c01 --- /dev/null +++ b/src/nix/flake-new.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Create a flake using the default template in the directory `hello`: + + ```console + # nix flake new hello + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template in the directory `hello`: + + ```console + # nix flake new hello -t templates#trivial + ``` + +# Description + +This command creates a flake in the directory `dest-dir`, which must +not already exist. It's equivalent to: + +```console +# mkdir dest-dir +# cd dest-dir +# nix flake init +``` + +)"" diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md new file mode 100644 index 000000000..1a42c44a0 --- /dev/null +++ b/src/nix/flake-show.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Show the output attributes provided by the `patchelf` flake: + + ```console + github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e + ├───checks + │ ├───aarch64-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux + │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + ├───defaultPackage + │ ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux: package 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b' + ├───hydraJobs + │ ├───build + │ │ ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b' + │ ├───release: derivation 'patchelf-0.12.20201207.f34751b' + │ └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b' + └───overlay: Nixpkgs overlay + ``` + +# Description + +This command shows the output attributes provided by the flake +specified by flake reference *flake-url*. These are the top-level +attributes in the `outputs` of the flake, as well as lower-level +attributes for some standard outputs (e.g. `packages` or `checks`). + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md new file mode 100644 index 000000000..a2ffedd2a --- /dev/null +++ b/src/nix/flake-update.md @@ -0,0 +1,53 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake update --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +* Recreate the lock file (i.e. update all inputs) and commit the new + lock file: + + ```console + # nix flake update --recreate-lock-file --commit-lock-file + … + warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Note that every command that operates on a flake will +also update the lock file if needed, and supports the same +flags. Therefore, + +```console +# nix flake update --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +> **Note** +> +> This command does *not* update locks that are already present unless +> you explicitly ask for it using `--update-input` or +> `--recreate-lock-file`. Thus, if the lock file already has locks for +> every input, then `nix flake update` (without arguments) does +> nothing. + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 066430c5d..2b91faa64 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -104,6 +104,13 @@ struct CmdFlakeUpdate : FlakeCommand return "update flake lock file"; } + std::string doc() override + { + return + #include "flake-update.md" + ; + } + void run(nix::ref store) override { /* Use --refresh by default for 'nix flake update'. */ @@ -134,6 +141,13 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON return "list info about a given flake"; } + std::string doc() override + { + return + #include "flake-info.md" + ; + } + void run(nix::ref store) override { auto flake = getFlake(); @@ -153,6 +167,13 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON return "list flake inputs"; } + std::string doc() override + { + return + #include "flake-list-inputs.md" + ; + } + void run(nix::ref store) override { auto flake = lockFlake(); @@ -211,6 +232,13 @@ struct CmdFlakeCheck : FlakeCommand return "check whether the flake evaluates and run its tests"; } + std::string doc() override + { + return + #include "flake-check.md" + ; + } + void run(nix::ref store) override { settings.readOnlyMode = !build; @@ -631,22 +659,11 @@ struct CmdFlakeInit : CmdFlakeInitCommon return "create a flake in the current directory from a template"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a flake using the default template:", - "nix flake init" - }, - Example{ - "To see available templates:", - "nix flake show templates" - }, - Example{ - "To create a flake from a specific template:", - "nix flake init -t templates#nixos-container" - }, - }; + return + #include "flake-init.md" + ; } CmdFlakeInit() @@ -662,6 +679,13 @@ struct CmdFlakeNew : CmdFlakeInitCommon return "create a flake in the specified directory from a template"; } + std::string doc() override + { + return + #include "flake-new.md" + ; + } + CmdFlakeNew() { expectArgs({ @@ -681,6 +705,13 @@ struct CmdFlakeClone : FlakeCommand return "clone flake repository"; } + std::string doc() override + { + return + #include "flake-clone.md" + ; + } + CmdFlakeClone() { addFlag({ @@ -720,22 +751,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun return "copy a flake and all its inputs to a store"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy the dwarffs flake and its dependencies to a binary cache:", - "nix flake archive --to file:///tmp/my-cache dwarffs" - }, - Example{ - "To fetch the dwarffs flake and its dependencies to the local Nix store:", - "nix flake archive dwarffs" - }, - Example{ - "To print the store paths of the flake sources of NixOps without fetching them:", - "nix flake archive --json --dry-run nixops" - }, - }; + return + #include "flake-archive.md" + ; } void run(nix::ref store) override @@ -797,6 +817,13 @@ struct CmdFlakeShow : FlakeCommand return "show the outputs provided by a flake"; } + std::string doc() override + { + return + #include "flake-show.md" + ; + } + void run(nix::ref store) override { auto state = getEvalState(); @@ -955,6 +982,13 @@ struct CmdFlake : NixMultiCommand return "manage Nix flakes"; } + std::string doc() override + { + return + #include "flake.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/flake.md b/src/nix/flake.md new file mode 100644 index 000000000..440c45dd1 --- /dev/null +++ b/src/nix/flake.md @@ -0,0 +1,566 @@ +R""( + +# Description + +`nix flake` provides subcommands for creating, modifying and querying +*Nix flakes*. Flakes are the unit for packaging Nix code in a +reproducible and discoverable way. They can have dependencies on other +flakes, making it possible to have multi-repository Nix projects. + +A flake is a filesystem tree (typically fetched from a Git repository +or a tarball) that contains a file named `flake.nix` in the root +directory. `flake.nix` specifies some metadata about the flake such as +dependencies (called *inputs*), as well as its *outputs* (the Nix +values such as packages or NixOS modules provided by the flake). + +# Flake references + +Flake references (*flakerefs*) are a way to specify the location of a +flake. These have two different forms: + +* An attribute set representation, e.g. + + ```nix + { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + } + ``` + + The only required attribute is `type`. The supported types are + listed below. + +* A URL-like syntax, e.g. + + ``` + github:NixOS/nixpkgs + ``` + + These are used on the command line as a more convenient alternative + to the attribute set representation. For instance, in the command + + ```console + # nix build github:NixOS/nixpkgs#hello + ``` + + `github:NixOS/nixpkgs` is a flake reference (while `hello` is an + output attribute). They are also allowed in the `inputs` attribute + of a flake, e.g. + + ```nix + inputs.nixpkgs.url = github:NixOS/nixpkgs; + ``` + + is equivalent to + + ```nix + inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }; + ``` + +## Examples + +Here are some examples of flake references in their URL-like representation: + +* `.`: The flake in the current directory. +* `/home/alice/src/patchelf`: A flake in some other directory. +* `nixpkgs`: The `nixpkgs` entry in the flake registry. +* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` + entry in the flake registry, with its Git revision overriden to a + specific value. +* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` + repository on GitHub. +* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the + `nixpkgs` repository. +* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A + specific revision of the `nixpkgs` repository. +* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory + of a GitHub repository. +* `git+https://github.com/NixOS/patchelf`: A Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master`: A specific + branch of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: + A specific branch *and* revision of a Git repository. +* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball + flake. + +## Flake reference attributes + +The following generic flake reference attributes are supported: + +* `dir`: The subdirectory of the flake in which `flake.nix` is + located. This parameter enables having multiple flakes in a + repository or tarball. The default is the root directory of the + flake. + +* `narHash`: The hash of the NAR serialisation (in SRI format) of the + contents of the flake. This is useful for flake types such as + tarballs that lack a unique content identifier such as a Git commit + hash. + +In addition, the following attributes are common to several flake +reference types: + +* `rev`: A Git or Mercurial commit hash. + +* `ref`: A Git or Mercurial branch or tag name. + +Finally, some attribute are typically not specified by the user, but +can occur in *locked* flake references and are available to Nix code: + +* `revCount`: The number of ancestors of the commit `rev`. + +* `lastModified`: The timestamp (in seconds since the Unix epoch) of + the last modification of this version of the flake. For + Git/Mercurial flakes, this is the commit time of commit *rev*, while + for tarball flakes, it's the most recent timestamp of any file + inside the tarball. + +## Types + +Currently the `type` attribute can be one of the following: + +* `path`: arbitrary local directories, or local Git trees. The + required attribute `path` specifies the path of the flake. The URL + form is + + ``` + [path:](\?` (see below), + except that the `dir` parameter is derived automatically. For + example, if `/foo/bar` is a Git repository, then the flake reference + `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`. + + If the directory is not inside a Git repository, then the flake + contents is the entire contents of *path*. + + *path* generally must be an absolute path. However, on the command + line, it can be a relative path (e.g. `.` or `./foo`) which is + interpreted as relative to the current directory. In this case, it + must start with `.` to avoid ambiguity with registry lookups + (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative + path). + +* `git`: Git repositories. The location of the repository is specified + by the attribute `url`. + + They have the URL form + + ``` + git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + ``` + + The `ref` attribute defaults to `master`. + + The `rev` attribute must denote a commit that exists in the branch + or tag specified by the `ref` attribute, since Nix doesn't do a full + clone of the remote repository by default (and the Git protocol + doesn't allow fetching a `rev` without a known `ref`). The default + is the commit currently pointed to by `ref`. + + For example, the following are valid Git flake references: + + * `git+https://example.org/my/repo` + * `git+https://example.org/my/repo?dir=flake1` + * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` + * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` + * `git+file:///home/my-user/some-repo/some-repo` + +* `mercurial`: Mercurial repositories. The URL form is similar to the + `git` type, except that the URL schema must be one of `hg+http`, + `hg+https`, `hg+ssh` or `hg+file`. + +* `tarball`: Tarballs. The location of the tarball is specified by the + attribute `url`. + + In URL form, the schema must be `http://`, `https://` or `file://` + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` + or `.tar.bz2`. + +* `github`: A more efficient way to fetch repositories from + GitHub. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + These are downloaded as tarball archives, rather than + through Git. This is often much faster and uses less disk space + since it doesn't require fetching the entire history of the + repository. On the other hand, it doesn't allow incremental fetching + (but full downloads are often faster than incremental fetches!). + + The URL syntax for `github` flakes is: + + ``` + github:/(/)?(\?)? + ``` + + `` specifies the name of a branch or tag (`ref`), or a + commit hash (`rev`). Note that unlike Git, GitHub allows fetching by + commit hash without specifying a branch or tag. + + Some examples: + + * `github:edolstra/dwarffs` + * `github:edolstra/dwarffs/unstable` + * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + +* `indirect`: Indirections through the flake registry. These have the + form + + ``` + [flake:](/(/rev)?)? + ``` + + These perform a lookup of `` in the flake registry. or + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + +# Flake format + +As an example, here is a simple `flake.nix` that depends on the +Nixpkgs flake and provides a single package (i.e. an installable +derivation): + +```nix +{ + description = "A flake for building Hello World"; + + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + + outputs = { self, nixpkgs }: { + + defaultPackage.x86_64-linux = + # Notice the reference to nixpkgs here. + with import nixpkgs { system = "x86_64-linux"; }; + stdenv.mkDerivation { + name = "hello"; + src = self; + buildPhase = "gcc -o hello ./hello.c"; + installPhase = "mkdir -p $out/bin; install -t $out/bin hello"; + }; + + }; +} +``` + +The following attributes are supported in `flake.nix`: + +* `description`: A short, one-line description of the flake. + +* `inputs`: An attrset specifying the dependencies of the flake + (described below). + +* `outputs`: A function that, given an attribute set containing the + outputs of each of the input flakes keyed by their identifier, + yields the Nix values provided by this flake. Thus, in the example + above, `inputs.nixpkgs` contains the result of the call to the + `outputs` function of the `nixpkgs` flake. + + In addition to the outputs of each input, each input in `inputs` + also contains some metadata about the inputs. These are: + + * `outPath`: The path in the Nix store of the flake's source tree. + + * `rev`: The commit hash of the flake's repository, if applicable. + + * `revCount`: The number of ancestors of the revision `rev`. This is + not available for `github` repositories, since they're fetched as + tarballs rather than as Git repositories. + + * `lastModifiedDate`: The commit time of the revision `rev`, in the + format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`, + this is available for both Git and GitHub repositories, so it's + useful for generating (hopefully) monotonically increasing version + strings. + + * `lastModified`: The commit time of the revision `rev` as an integer + denoting the number of seconds since 1970. + + * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of + the flake's source tree. + + The value returned by the `outputs` function must be an attribute + set. The attributes can have arbitrary values; however, various + `nix` subcommands require specific attributes to have a specific + value (e.g. `packages.x86_64-linux` must be an attribute set of + derivations built for the `x86_64-linux` platform). + +## Flake inputs + +The attribute `inputs` specifies the dependencies of a flake, as an +attrset mapping input names to flake references. For example, the +following specifies a dependency on the `nixpkgs` and `import-cargo` +repositories: + +```nix +# A GitHub repository. +inputs.import-cargo = { + type = "github"; + owner = "edolstra"; + repo = "import-cargo"; +}; + +# An indirection through the flake registry. +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Alternatively, you can use the URL-like syntax: + +```nix +inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.nixpkgs.url = "nixpkgs"; +``` + +Each input is fetched, evaluated and passed to the `outputs` function +as a set of attributes with the same name as the corresponding +input. The special input named `self` refers to the outputs and source +tree of *this* flake. Thus, a typical `outputs` function looks like +this: + +```nix +outputs = { self, nixpkgs, import-cargo }: { + ... outputs ... +}; +``` + +It is also possible to omit an input entirely and *only* list it as +expected function argument to `outputs`. Thus, + +```nix +outputs = { self, nixpkgs }: ...; +``` + +without an `inputs.nixpkgs` attribute is equivalent to + +```nix +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Repositories that don't contain a `flake.nix` can also be used as +inputs, by setting the input's `flake` attribute to `false`: + +```nix +inputs.grcov = { + type = "github"; + owner = "mozilla"; + repo = "grcov"; + flake = false; +}; + +outputs = { self, nixpkgs, grcov }: { + packages.x86_64-linux.grcov = stdenv.mkDerivation { + src = grcov; + ... + }; +}; +``` + +Transitive inputs can be overriden from a `flake.nix` file. For +example, the following overrides the `nixpkgs` input of the `nixops` +input: + +```nix +inputs.nixops.inputs.nixpkgs = { + type = "github"; + owner = "my-org"; + repo = "nixpkgs"; +}; +``` + +It is also possible to "inherit" an input from another input. This is +useful to minimize flake dependencies. For example, the following sets +the `nixpkgs` input of the top-level flake to be equal to the +`nixpkgs` input of the `dwarffs` input of the top-level flake: + +```nix +inputs.nixops.follows = "dwarffs/nixpkgs"; +``` + +The value of the `follows` attribute is a `/`-separated sequence of +input names denoting the path of inputs to be followed from the root +flake. + +Overrides and `follows` can be combined, e.g. + +```nix +inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs"; +``` + +sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs` +input of `dwarffs`. It is worth noting, however, that it is generally +not useful to eliminate transitive `nixpkgs` flake inputs in this +way. Most flakes provide their functionality through Nixpkgs overlays +or NixOS modules, which are composed into the top-level flake's +`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant. + +# Lock files + +Inputs specified in `flake.nix` are typically "unlocked" in the sense +that they don't specify an exact revision. To ensure reproducibility, +Nix will automatically generate and use a *lock file* called +`flake.lock` in the flake's directory. The lock file contains a graph +structure isomorphic to the graph of dependencies of the root +flake. Each node in the graph (except the root node) maps the +(usually) unlocked input specifications in `flake.nix` to locked input +specifications. Each node also contains some metadata, such as the +dependencies (outgoing edges) of the node. + +For example, if `flake.nix` has the inputs in the example above, then +the resulting lock file might be: + +```json +{ + "version": 7, + "root": "n1", + "nodes": { + "n1": { + "inputs": { + "nixpkgs": "n2", + "import-cargo": "n3", + "grcov": "n4" + } + }, + "n2": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013", + "type": "github", + "lastModified": 1580555482, + "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8=" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "n3": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "import-cargo", + "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422", + "type": "github", + "lastModified": 1567183309, + "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc=" + }, + "original": { + "owner": "edolstra", + "repo": "import-cargo", + "type": "github" + } + }, + "n4": { + "inputs": {}, + "locked": { + "owner": "mozilla", + "repo": "grcov", + "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e", + "type": "github", + "lastModified": 1580729070, + "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI=" + }, + "original": { + "owner": "mozilla", + "repo": "grcov", + "type": "github" + }, + "flake": false + } + } +} +``` + +This graph has 4 nodes: the root flake, and its 3 dependencies. The +nodes have arbitrary labels (e.g. `n1`). The label of the root node of +the graph is specified by the `root` attribute. Nodes contain the +following fields: + +* `inputs`: The dependencies of this node, as a mapping from input + names (e.g. `nixpkgs`) to node labels (e.g. `n2`). + +* `original`: The original input specification from `flake.lock`, as a + set of `builtins.fetchTree` arguments. + +* `locked`: The locked input specification, as a set of + `builtins.fetchTree` arguments. Thus, in the example above, when we + build this flake, the input `nixpkgs` is mapped to revision + `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs` + repository on GitHub. + + It also includes the attribute `narHash`, specifying the expected + contents of the tree in the Nix store (as computed by `nix + hash-path`), and may include input-type-specific attributes such as + the `lastModified` or `revCount`. The main reason for these + attributes is to allow flake inputs to be substituted from a binary + cache: `narHash` allows the store path to be computed, while the + other attributes are necessary because they provide information not + stored in the store path. + +* `flake`: A Boolean denoting whether this is a flake or non-flake + dependency. Corresponds to the `flake` attribute in the `inputs` + attribute in `flake.nix`. + +The `original` and `locked` attributes are omitted for the root +node. This is because we cannot record the commit hash or content hash +of the root flake, since modifying `flake.lock` will invalidate these. + +The graph representation of lock files allows circular dependencies +between flakes. For example, here are two flakes that reference each +other: + +```nix +{ + inputs.b = ... location of flake B ...; + # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is + # *this* 'a'. + inputs.b.inputs.a.follows = ""; + outputs = { self, b }: { + foo = 123 + b.bar; + xyzzy = 1000; + }; +} +``` + +and + +```nix +{ + inputs.a = ... location of flake A ...; + inputs.a.inputs.b.follows = ""; + outputs = { self, a }: { + bar = 456 + a.xyzzy; + }; +} +``` + +Lock files transitively lock direct as well as indirect +dependencies. That is, if a lock file exists and is up to date, Nix +will not look at the lock files of dependencies. However, lock file +generation itself *does* use the lock files of dependencies by +default. + +)"" From 1047cb1e53358755cb12a5361cfc99118ed7e159 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 13:21:31 +0100 Subject: [PATCH 063/113] Command: Remove examples() --- doc/manual/generate-manpage.nix | 5 ----- src/libutil/args.cc | 30 ------------------------------ src/libutil/args.hh | 14 -------------- 3 files changed, 49 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 9f30c8fbc..c2c748464 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -20,11 +20,6 @@ let (attrNames def.commands)) + "\n" else "") - + (if def.examples or [] != [] - then - "# Examples\n\n" - + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) - else "") + (if def ? doc then def.doc + "\n\n" else "") diff --git a/src/libutil/args.cc b/src/libutil/args.cc index a929ea5ac..fb5cb80fb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -353,36 +353,6 @@ void printTable(std::ostream & out, const Table2 & table) } } -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } -} - -nlohmann::json Command::toJSON() -{ - auto exs = nlohmann::json::array(); - - for (auto & example : examples()) { - auto ex = nlohmann::json::object(); - ex["description"] = example.description; - ex["command"] = chomp(stripIndentation(example.command)); - exs.push_back(std::move(ex)); - } - - auto res = Args::toJSON(); - res["examples"] = std::move(exs); - return res; -} - MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 68bbbb4f7..6ed541a32 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -228,25 +228,11 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; - struct Example - { - std::string description; - std::string command; - }; - - typedef std::list Examples; - - virtual Examples examples() { return Examples(); } - typedef int Category; static constexpr Category catDefault = 0; virtual Category category() { return catDefault; } - - void printHelp(const string & programName, std::ostream & out) override; - - nlohmann::json toJSON() override; }; typedef std::map()>> Commands; From 26e502ceb567e48f3eb28fb1848430ab69cb7606 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 16:41:07 +0100 Subject: [PATCH 064/113] Add TODO --- src/nix/bundle.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index c183a170d..5e2298376 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -29,4 +29,8 @@ for more details. > > This command only works on Linux. +# Bundler definitions + +TODO + )"" From 5178211e963fa111f84c4881b22cc506d5254fde Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Dec 2020 18:33:42 +0100 Subject: [PATCH 065/113] Add 'nix' manpage --- src/nix/main.cc | 7 +++ src/nix/nix.md | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 src/nix/nix.md diff --git a/src/nix/main.cc b/src/nix/main.cc index afe7cb8d7..b2406fafe 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -184,6 +184,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { return "a tool for reproducible and declarative configuration management"; } + + std::string doc() override + { + return + #include "nix.md" + ; + } }; static void showHelp(std::vector subcommand) diff --git a/src/nix/nix.md b/src/nix/nix.md new file mode 100644 index 000000000..d10de7c01 --- /dev/null +++ b/src/nix/nix.md @@ -0,0 +1,119 @@ +R""( + +# Examples + +* Create a new flake: + + ```console + # nix flake new hello + # cd hello + ``` + +* Build the flake in the current directory: + + ```console + # nix build + # ./result/bin/hello + Hello, world! + ``` + +* Run the flake in the current directory: + + ```console + # nix run + Hello, world! + ``` + +* Start a development shell for hacking on this flake: + + ```console + # nix develop + # unpackPhase + # cd hello-* + # configurePhase + # buildPhase + # ./hello + Hello, world! + # installPhase + # ../outputs/out/bin/hello + Hello, world! + ``` + +# Description + +Nix is a tool for building software, configurations and other +artifacts in a reproducible and declarative way. For more information, +see the [Nix homepage](https://nixos.org/) or the [Nix +manual](https://nixos.org/manual/nix/stable/). + +# Installables + +Many `nix` subcommands operate on one or more *installables*. These are +command line arguments that represent something that can be built in +the Nix store. Here are the recognised types of installables: + +* **Flake output attributes**: `nixpkgs#hello` + + These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a + flake reference and *attrpath* is an optional attribute path. For + more information on flakes, see [the `nix flake` manual + page](./nix3-flake.md). Flake references are most commonly a flake + identifier in the flake registry (e.g. `nixpkgs`) or a path + (e.g. `/path/to/my-flake` or `.`). + + If *attrpath* is omitted, Nix tries some default values; for most + subcommands, the default is `defaultPackage.`*system* + (e.g. `defaultPackage.x86_64-linux`), but some subcommands have + other defaults. If *attrpath* *is* specified, *attrpath* is + interpreted as relative to one or more prefixes; for most + subcommands, these are `packages.`*system*, + `legacyPackages.*system*` and the empty prefix. Thus, on + `x86_64-linux` `nix build nixpkgs#hello` will try to build the + attributes `packages.x86_64-linux.hello`, + `legacyPackages.x86_64-linux.hello` and `hello`. + +* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + + These are paths inside the Nix store, or symlinks that resolve to a + path in the Nix store. + +* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` + + Store derivations are store paths with extension `.drv` and are a + low-level representation of a build-time dependency graph used + internally by Nix. By default, if you pass a store derivation to a + `nix` subcommand, it will operate on the *output paths* of the + derivation. For example, `nix path-info` prints information about + the output paths: + + ```console + # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv + [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + ``` + + If you want to operate on the store derivation itself, pass the + `--derivation` flag. + +* **Nix attributes**: `--file /path/to/nixpkgs hello` + + When the `-f` / `--file` *path* option is given, installables are + interpreted as attribute paths referencing a value returned by + evaluating the Nix file *path*. + +* **Nix expressions**: `--expr '(import {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. + + When the `--expr` option is given, all installables are interpreted + as Nix expressions. You may need to specify `--impure` if the + expression references impure inputs (such as ``). + +For most commands, if no installable is specified, the default is `.`, +i.e. Nix will operate on the default flake output attribute of the +flake in the current directory. + +# Nix stores + +Most `nix` subcommands operate on a *Nix store*. + +TODO: list store types, options + +)"" From d4870462f8f539adeaa6dca476aff6f1f31e1981 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Dec 2020 14:16:06 -0600 Subject: [PATCH 066/113] Cast variants fully for libc++10 libc++10 seems to be stricter on what it allows in variant conversion. I'm not sure what the rules are here, but this is the minimal change needed to get through the compilation errors. --- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/attrs.cc | 2 +- src/libfetchers/github.cc | 4 ++-- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..539ba71f3 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -394,7 +394,7 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; else if (v.type == tPath) - cachedValue = {root->db->setString(getKey(), v.path), v.path}; + cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; else if (v.type == tBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type == tAttrs) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..1360ade39 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -104,7 +104,7 @@ static void fetchTree( else if (attr.value->type == tBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type == tInt) - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 720b19fcd..17fc4041f 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -11,7 +11,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) for (auto & i : json.items()) { if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), i.value().get()); else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..db1ced5d6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -195,14 +195,14 @@ struct GitArchiveInputScheme : InputScheme auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", lastModified); + input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", lastModified} + {"lastModified", uint64_t(lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 07a51059d..0eb401e10 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, - {"revCount", (int64_t) revCount}, + {"revCount", (uint64_t) revCount}, }); if (!_input.getRev()) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 8c0f20475..56c014a8c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -152,7 +152,7 @@ std::pair downloadTarball( } Attrs infoAttrs({ - {"lastModified", lastModified}, + {"lastModified", uint64_t(lastModified)}, {"etag", res.etag}, }); From 9d3aad7b92762973081738a8bc6a562fda45c341 Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:43:22 +0000 Subject: [PATCH 067/113] Update URL where bzip2 can be obtained --- doc/manual/src/installation/prerequisites-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 69b7c5a5e..6825af707 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -30,7 +30,7 @@ have bzip2 installed, including development headers and libraries. If your distribution does not provide these, you can obtain bzip2 from - . + . - `liblzma`, which is provided by XZ Utils. If your distribution does not provide this, you can get it from . From f1e9bda9d1ece9be8c78f5c5345c3adb299bc4aa Mon Sep 17 00:00:00 2001 From: Sevan Janiyan Date: Fri, 25 Dec 2020 01:48:21 +0000 Subject: [PATCH 068/113] Update URL where bzip2 can be obtained --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index daf378997..c1bfc9b53 100644 --- a/configure.ac +++ b/configure.ac @@ -174,9 +174,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"] # Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed From 6262a703636000e525d5c1b877ac28d604a493f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Dec 2020 17:21:19 +0100 Subject: [PATCH 069/113] scanForReferences: Remove misleading comment References have always been determined only by the hash part, not the name or the store prefix. Fixes #4396. --- src/libstore/references.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index eb117b5ba..39c4970c6 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee, TeeSink sink { refsSink, toTee }; std::map backMap; - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ for (auto & i : refs) { auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); From 093de16223b8b93d803e4cd1cc1d3945cb3dfeb1 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 28 Dec 2020 09:30:14 -0800 Subject: [PATCH 070/113] README: fix link to hacking guide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11fe5f932..4686010ef 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From 64904b9d5d32c4201aaf462ae82b736f33785793 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 28 Dec 2020 19:40:04 -0600 Subject: [PATCH 071/113] Fixup --- src/libexpr/eval-cache.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0f84944cd..98d91c905 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -393,9 +393,9 @@ Value & AttrCursor::forceValue() if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == nPath) + else if (v.type() == nPath) cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; - else if (v.type == nBool) + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nAttrs) ; // FIXME: do something? diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e64e3fbb8..ab80be2d3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -103,7 +103,7 @@ static void fetchTree( addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); - else if (attr.value->type == nInt) + else if (attr.value->type() == nInt) attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", From d27eb0ef573b4739967119448779da4a8b2a2cbf Mon Sep 17 00:00:00 2001 From: David McFarland Date: Wed, 30 Dec 2020 16:20:03 -0400 Subject: [PATCH 072/113] Fix insufficent attribute capacity in user profile --- src/nix-env/user-env.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 87387e794..168ac492b 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + DrvInfo::Outputs outputs = i.queryOutputs(true); + StringSet metaNames = i.queryMetaNames(); Value & v(*state.allocValue()); manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); + state.mkAttrs(v, 7 + outputs.size()); mkString(*state.allocAttr(v, state.sType), "derivation"); mkString(*state.allocAttr(v, state.sName), i.queryName()); @@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; @@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); + state.mkAttrs(vMeta, metaNames.size()); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) continue; From e069ddf3258b7eab2074639cade03ba03b0a03a4 Mon Sep 17 00:00:00 2001 From: Sam Lidder Date: Thu, 31 Dec 2020 20:17:37 -0500 Subject: [PATCH 073/113] Fix `configure` error in introduction doc --- doc/manual/src/introduction.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index f01fe7b38..d68445c95 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -165,10 +165,10 @@ You’re then dropped into a shell where you can edit, build and test the package: ```console -[nix-shell]$ tar xf $src +[nix-shell]$ unpackPhase [nix-shell]$ cd pan-* -[nix-shell]$ ./configure -[nix-shell]$ make +[nix-shell]$ configurePhase +[nix-shell]$ buildPhase [nix-shell]$ ./pan/gui/pan ``` From 988dd0a65f562741708f6a7a7a44e333d6a5b205 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Tue, 5 Jan 2021 02:06:25 +0000 Subject: [PATCH 074/113] Fix conversion from JSON to fetch attributes It appears as through the fetch attribute, which is simply a variant with 3 elements, implicitly converts boolean arguments to integers. One must use Explicit to correctly populate it with a boolean. This was missing from the implementation, and resulted in clearly boolean JSON fields being treated as numbers. --- src/libfetchers/attrs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 17fc4041f..a565d19d4 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -15,7 +15,7 @@ Attrs jsonToAttrs(const nlohmann::json & json) else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_boolean()) - attrs.emplace(i.key(), i.value().get()); + attrs.emplace(i.key(), Explicit { i.value().get() }); else throw Error("unsupported input attribute type in lock file"); } From 8af4f886e212346afdd1d40789f96f1321da96c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 11:47:29 +0100 Subject: [PATCH 075/113] Fix deadlock in LocalStore::addSignatures() Fixes #4367. --- src/libstore/local-store.cc | 95 +++++++++++++++++++------------------ src/libstore/local-store.hh | 2 + 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..702e7b136 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -736,57 +736,62 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - callback(retrySQLite>([&]() { + callback(retrySQLite>([&]() { auto state(_state.lock()); - - /* Get the path info. */ - auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); - - if (!useQueryPathInfo.next()) - return std::shared_ptr(); - - auto id = useQueryPathInfo.getInt(0); - - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } - - auto info = std::make_shared(path, narHash); - - info->id = id; - - info->registrationTime = useQueryPathInfo.getInt(2); - - auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); - - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); - - info->ultimate = useQueryPathInfo.getInt(5) == 1; - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); - - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); - - /* Get the references. */ - auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); - - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - - return info; + return queryPathInfoInternal(*state, path); })); } catch (...) { callback.rethrow(); } } +std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) +{ + /* Get the path info. */ + auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); + + if (!useQueryPathInfo.next()) + return std::shared_ptr(); + + auto id = useQueryPathInfo.getInt(0); + + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } + + auto info = std::make_shared(path, narHash); + + info->id = id; + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + if (s) info->deriver = parseStorePath(s); + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + if (s) info->sigs = tokenizeString(s, " "); + + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + if (s) info->ca = parseContentAddressOpt(s); + + /* Get the references. */ + auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); + + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + + return info; +} + + /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { @@ -1608,7 +1613,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); + auto info = std::const_pointer_cast(queryPathInfoInternal(*state, storePath)); info->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..6d29c5960 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,6 +235,8 @@ private: void verifyPath(const Path & path, const StringSet & store, PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); From 146af4ee9bb03968a7322a1ac70dc60c8d5a35e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 16:43:09 +0100 Subject: [PATCH 076/113] Move sodium_init() call --- src/libmain/shared.cc | 9 +++++++++ src/nix-store/nix-store.cc | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2247aeca4..e9f067e35 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,6 +18,10 @@ #include +#if HAVE_SODIUM +#include +#endif + namespace nix { @@ -126,6 +130,11 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif +#if HAVE_SODIUM + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); +#endif + loadConfFile(); startSignalHandlerThread(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54394e921..e1ccece99 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -981,9 +981,6 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string publicKeyFile = *i++; #if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:04:46 +0100 Subject: [PATCH 077/113] crypto.cc: API cleanup and add generate() / to_string() methods --- src/libstore/crypto.cc | 33 ++++++++++++++++++++++++++------- src/libstore/crypto.hh | 24 ++++++++++++++++-------- src/nix-store/nix-store.cc | 17 +++-------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..135ced277 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -8,15 +8,15 @@ namespace nix { -static std::pair split(const string & s) +static std::pair split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,7 +29,12 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) +std::string Key::to_string() const +{ + return name + ":" + base64Encode(key); +} + +SecretKey::SecretKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s) } #endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { #if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; @@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const #endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig, #if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘:’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e1ccece99..e43788bc3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -19,10 +19,6 @@ #include #include -#if HAVE_SODIUM -#include -#endif - namespace nix_store { @@ -980,18 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string secretKeyFile = *i++; string publicKeyFile = *i++; -#if HAVE_SODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); + auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif + writeFile(secretKeyFile, secretKey.to_string()); } From 9374c2baeabe45a22e4b8746dc97f5ce4f030184 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:41:16 +0100 Subject: [PATCH 078/113] Add commands for generating secret/public keys --- src/nix/hash.cc | 5 -- src/nix/key-convert-secret-to-public.md | 19 ++++++ src/nix/key-generate-secret.md | 48 ++++++++++++++ src/nix/sigs.cc | 87 +++++++++++++++++++++++++ tests/binary-cache.sh | 12 ++-- 5 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/nix/key-convert-secret-to-public.md create mode 100644 src/nix/key-generate-secret.md diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 101b67e6a..6fd791f41 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,11 +132,6 @@ struct CmdHash : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..6ff1e1c9b --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign-paths --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 37b8a6712..b2e598ad5 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -141,3 +141,90 @@ struct CmdSignPaths : StorePathsCommand }; static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); + +#if HAVE_SODIUM +struct CmdKeyGenerateSecret : Command +{ + std::optional keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "identifier of the key (e.g. `cache.example.org-1`)", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref(); }}, + {"convert-secret-to-public", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand("key"); +#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..1a06404ed 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -131,14 +131,14 @@ if [ -n "$HAVE_SODIUM" ]; then clearCache clearCacheCache -declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) -publicKey="$(cat $TEST_ROOT/pk1)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 +publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) -res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2)) -badKey="$(cat $TEST_ROOT/pk2)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 +badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) -res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) -otherKey="$(cat $TEST_ROOT/pk3)" +nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 +otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:56:53 +0100 Subject: [PATCH 079/113] Make sodium a required dependency --- Makefile.config.in | 1 - configure.ac | 6 +----- perl/Makefile.config.in | 1 - perl/configure.ac | 6 +----- perl/lib/Nix/Store.xs | 10 ---------- src/libmain/shared.cc | 6 ------ src/libstore/crypto.cc | 29 ----------------------------- src/nix/sigs.cc | 2 -- tests/binary-cache.sh | 4 ---- tests/common.sh.in | 1 - 10 files changed, 2 insertions(+), 64 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 3845b3be0..d1e59e4e7 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -HAVE_SODIUM = @HAVE_SODIUM@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ diff --git a/configure.ac b/configure.ac index c1bfc9b53..2047ed8d2 100644 --- a/configure.ac +++ b/configure.ac @@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA ]) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index c87d4817e..eccfbd9f6 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -2,7 +2,6 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -HAVE_SODIUM = @HAVE_SODIUM@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ diff --git a/perl/configure.ac b/perl/configure.ac index 255744afd..85183c005 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). perlFlags="-I$perllibdir" diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e3b7d389..ad9042a2a 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -14,9 +14,7 @@ #include "util.hh" #include "crypto.hh" -#if HAVE_SODIUM #include -#endif using namespace nix; @@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32) SV * signString(char * secretKey_, char * msg) PPCODE: try { -#if HAVE_SODIUM auto sig = SecretKey(secretKey_).signDetached(msg); XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } @@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg) int checkSignature(SV * publicKey_, SV * sig_, char * msg) CODE: try { -#if HAVE_SODIUM STRLEN publicKeyLen; unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) @@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) throw Error("signature is not valid"); RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e9f067e35..6751a3744 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,9 +18,7 @@ #include -#if HAVE_SODIUM #include -#endif namespace nix { @@ -130,10 +128,8 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif -#if HAVE_SODIUM if (sodium_init() == -1) throw Error("could not initialise libsodium"); -#endif loadConfFile(); @@ -283,9 +279,7 @@ void printVersion(const string & programName) #if HAVE_BOEHMGC cfg.push_back("gc"); #endif -#if HAVE_SODIUM cfg.push_back("signed-caches"); -#endif std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 135ced277..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,9 +2,7 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include -#endif namespace nix { @@ -37,70 +35,46 @@ std::string Key::to_string() const SecretKey::SecretKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); -#endif } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -} -#endif - std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } SecretKey SecretKey::generate(std::string_view name) { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); -#else - noSodium(); -#endif } PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); auto key = publicKeys.find(std::string(ss.first)); @@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index b2e598ad5..14e2c9761 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -142,7 +142,6 @@ struct CmdSignPaths : StorePathsCommand static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); -#if HAVE_SODIUM struct CmdKeyGenerateSecret : Command { std::optional keyName; @@ -227,4 +226,3 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); -#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 1a06404ed..355a37d97 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -125,8 +125,6 @@ grep -q "copying path.*input-0" $TEST_ROOT/log grep -q "copying path.*top" $TEST_ROOT/log -if [ -n "$HAVE_SODIUM" ]; then - # Create a signed binary cache. clearCache clearCacheCache @@ -181,8 +179,6 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" -fi # HAVE_LIBSODIUM - unset _NIX_FORCE_HTTP diff --git a/tests/common.sh.in b/tests/common.sh.in index 5e00d64f1..5489c0c44 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -34,7 +34,6 @@ coreutils=@coreutils@ export dot=@dot@ export SHELL="@bash@" export PAGER=cat -export HAVE_SODIUM="@HAVE_SODIUM@" export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: [PATCH 080/113] Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 232 --------------- src/nix/local.mk | 1 - src/nix/prefetch.cc | 352 +++++++++++++++++++++++ src/nix/store-prefetch-file.md | 32 +++ src/nix/store-prefetch-tarball.md | 31 ++ 6 files changed, 416 insertions(+), 234 deletions(-) delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc create mode 100644 src/nix/prefetch.cc create mode 100644 src/nix/store-prefetch-file.md create mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include - -#include -#include -#include - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional expectedHash; - Hash hash(ht); - std::optional storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md new file mode 100644 index 000000000..535d7e022 --- /dev/null +++ b/src/nix/store-prefetch-tarball.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download a tarball and unpack it, unless it already exists in the + Nix store: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ + --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= + ``` + +# Description + +This command downloads a tarball or zip file from *url*, unpacks it, +and adds the unpacked tree to the Nix store. It prints out the +resulting store path and the NAR hash of that store path. + +The name component of the store path defaults to `source`, but this +can be overriden using `--name`. + +)"" From 93f1678ec60bcacfcc857f361b5f63e37c498eb4 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 01:53:57 +0000 Subject: [PATCH 081/113] Allow Flake inputs to accept boolean and integer attributes I believe that this makes it possible to do things like Git inputs with submodules, but it also likely applies to other input types from libfetchers. --- src/libexpr/flake/flake.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4f021570c..41c93bcaa 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,11 +120,16 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) + if (attr.value->type() == nString) { attrs.emplace(attr.name, attr.value->string.s); - else - throw TypeError("flake input attribute '%s' is %s while a string is expected", + } else if (attr.value->type() == nBool) { + attrs.emplace(attr.name, Explicit{ attr.value->boolean }); + } else if (attr.value->type() == nInt) { + attrs.emplace(attr.name, attr.value->integer); + } else { + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", attr.name, showType(*attr.value)); + } } } catch (Error & e) { e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); From ba0f841a078402f95cf93693c3749743c3ab6246 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 03:13:42 +0000 Subject: [PATCH 082/113] Use switch statement instead of sequence of ifs --- src/libexpr/flake/flake.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 41c93bcaa..9f1e4063f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,15 +120,19 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) { - attrs.emplace(attr.name, attr.value->string.s); - } else if (attr.value->type() == nBool) { - attrs.emplace(attr.name, Explicit{ attr.value->boolean }); - } else if (attr.value->type() == nInt) { - attrs.emplace(attr.name, attr.value->integer); - } else { - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", - attr.name, showType(*attr.value)); + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + attr.name, showType(*attr.value)); } } } catch (Error & e) { From 48a9be2aabf6620ceb00caf7c4c917e4e0a81446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 10:44:55 +0100 Subject: [PATCH 083/113] Remove mkIntFlag --- src/libutil/args.hh | 26 +++++++++++++++----------- src/nix/verify.cc | 10 +++++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6ed541a32..3e84ac64a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -68,8 +68,12 @@ protected: , arity(ArityAny) { } - template - Handler(T * dest) + Handler(std::string * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) : fun([=](std::vector ss) { *dest = ss[0]; }) , arity(1) { } @@ -79,6 +83,15 @@ protected: : fun([=](std::vector ss) { *dest = val; }) , arity(0) { } + + template + Handler(I * dest) + : fun([=](std::vector ss) { + if (!string2Int(ss[0], *dest)) + throw UsageError("'%s' is not an integer", ss[0]); + }) + , arity(1) + { } }; /* Flags. */ @@ -161,15 +174,6 @@ public: }); } - template - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag(shortName, longName, description, [=](I n) { - *dest = n; - }); - } - template void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 16d42349f..620109aac 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,6 +20,7 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + addFlag({ .longName = "substituter", .shortName = 's', @@ -27,7 +28,14 @@ struct CmdVerify : StorePathsCommand .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "require that each path has at least N valid signatures", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override From 1d4954e73e389d319416bf29e949b4b1cbc9ebd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:40:36 +0100 Subject: [PATCH 084/113] Remove mkFlag integer specialisation --- src/libmain/shared.cc | 16 +++++++++++++--- src/libutil/args.hh | 18 ------------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6751a3744..223020378 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -211,9 +211,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); + const std::string & description, const std::string & dest) + { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"n"}, + .handler = {[=](std::string s) { + unsigned int n; + if (!string2Int(s, n)) + throw UsageError("'%s' is not an integer", s); + settings.set(dest, std::to_string(n)); + }} }); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3e84ac64a..c54b0efaf 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -174,24 +174,6 @@ public: }); } - template - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {"N"}, - .handler = {[=](std::string s) { - I n; - if (!string2Int(s, n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); From 29a445840a4f01dfb1533806f8dfc28f7dc4bee9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:42:44 +0100 Subject: [PATCH 085/113] Remove unused mkFlag1 --- src/libutil/args.hh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c54b0efaf..62b9516d8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -143,19 +143,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {label}, - .handler = {[=](std::string s) { fun(s); }} - }); - } - void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:22:21 +0100 Subject: [PATCH 086/113] string2Int(): Return std::optional --- src/libexpr/attr-path.cc | 14 ++++++-------- src/libexpr/get-drvs.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 +++---- src/libstore/build/derivation-goal.cc | 4 +--- src/libstore/globals.cc | 8 ++++++-- src/libstore/local-store.cc | 4 +++- src/libstore/names.cc | 12 ++++++------ src/libstore/nar-info.cc | 8 ++++++-- src/libstore/profiles.cc | 11 +++++------ src/libstore/store-api.cc | 13 +++++++------ src/libutil/args.hh | 4 +++- src/libutil/config.cc | 4 +++- src/libutil/util.hh | 18 ++++++++++++------ src/nix-env/nix-env.cc | 19 +++++++++---------- src/nix/profile.cc | 5 ++--- 17 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2d37dcb7e..9dd557205 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -52,9 +52,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,7 +63,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { if (v->type() != nAttrs) throw TypeError( @@ -82,17 +80,17 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 32c115c12..1a3990ea1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -214,8 +214,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int(v->string.s)) + return *n; } return def; } @@ -228,8 +228,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float(v->string.s)) + return *n; } return def; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index bcb904c0d..d1003de57 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme if (name == "rev" || name == "narHash") input.attrs.insert_or_assign(name, value); else if (name == "revCount" || name == "lastModified") { - uint64_t n; - if (!string2Int(value, n)) + if (auto n = string2Int(value)) + input.attrs.insert_or_assign(name, *n); + else throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input.attrs.insert_or_assign(name, n); } else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 223020378..f1feeddd6 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,10 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - unsigned int n; - if (!string2Int(s, n)) + if (auto n = string2Int(s)) + settings.set(dest, std::to_string(*n)); + else throw UsageError("'%s' is not an integer", s); - settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index ffae5d796..38f627b44 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -70,10 +70,9 @@ template N getIntArg(const string & opt, s.resize(s.size() - 1); } } - N n; - if (!string2Int(s, n)) - throw UsageError("'%1%' requires an integer argument", opt); - return n * multiplier; + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%1%' requires an integer argument", opt); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index af3ab87a9..35f365795 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ad66ef8a8..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -228,8 +228,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702e7b136..c61f34275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int(c1); + auto n2 = string2Int(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..01e2fcc7b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -932,19 +932,20 @@ std::optional decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 62b9516d8..823d843aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,7 +87,9 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (!string2Int(ss[0], *dest)) + if (auto n = string2Int(ss[0])) + *dest = *n; + else throw UsageError("'%s' is not an integer", ss[0]); }) , arity(1) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7af3e7883..7467e5ac0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -230,7 +230,9 @@ template void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); - if (!string2Int(str, value)) + if (auto n = string2Int(str)) + value = *n; + else throw UsageError("setting '%s' has invalid value '%s'", name, str); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0f82bed78..7a4d5fe92 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -397,21 +397,27 @@ bool statusOk(int status); /* Parse a string into an integer. */ -template bool string2Int(const string & s, N & n) +template +std::optional string2Int(const std::string & s) { - if (string(s, 0, 1) == "-" && !std::numeric_limits::is_signed) - return false; + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return {}; std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } /* Parse a string into a float. */ -template bool string2Float(const string & s, N & n) +template +std::optional string2Float(const string & s) { std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c2e075ed..9963f05d9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - GenerationNumber dstGen; - if (!string2Int(opArgs.front(), dstGen)) + if (auto dstGen = string2Int(opArgs.front())) + switchGeneration(globals, *dstGen); + else throw UsageError("expected a generation number"); - - switchGeneration(globals, dstGen); } @@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - GenerationNumber max; - if (!string2Int(str_max, max) || max == 0) + auto max = string2Int(str_max); + if (!max || *max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { - GenerationNumber n; - if (!string2Int(i, n)) + if (auto n = string2Int(i)) + gens.insert(*n); + else throw UsageError("invalid generation number '%1%'", i); - gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d8d2b3a70..8cdd34a20 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,9 +209,8 @@ public: std::vector res; for (auto & s : _matchers) { - size_t n; - if (string2Int(s, n)) - res.push_back(n); + if (auto n = string2Int(s)) + res.push_back(*n); else if (store->isStorePath(s)) res.push_back(s); else From 17beae299d5e6bb511c453d0b9d0d7ef906b3d14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:51:19 +0100 Subject: [PATCH 087/113] Support binary unit prefixes in command line arguments --- src/libmain/shared.cc | 6 ++---- src/libmain/shared.hh | 17 +---------------- src/libutil/args.hh | 5 +---- src/libutil/util.hh | 28 +++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f1feeddd6..e797c2fb9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,8 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - if (auto n = string2Int(s)) - settings.set(dest, std::to_string(*n)); - else - throw UsageError("'%s' is not an integer", s); + auto n = string2IntWithUnitPrefix(s); + settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 38f627b44..edc7b5efa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -57,22 +57,7 @@ template N getIntArg(const string & opt, { ++i; if (i == end) throw UsageError("'%1%' requires an argument", opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%1%' requires an integer argument", opt); + return string2IntWithUnitPrefix(*i); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 823d843aa..3783bc84f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,10 +87,7 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (auto n = string2Int(ss[0])) - *dest = *n; - else - throw UsageError("'%s' is not an integer", ss[0]); + *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7a4d5fe92..ab0bd865a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,12 +401,34 @@ template std::optional string2Int(const std::string & s) { if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return {}; + return std::nullopt; std::istringstream str(s); N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; +} + +/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + 'T' denoting a binary unit prefix. */ +template +N string2IntWithUnitPrefix(std::string s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.resize(s.size() - 1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); } /* Parse a string into a float. */ @@ -417,7 +439,7 @@ std::optional string2Float(const string & s) N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; } From e21aee58f6dd7785df50d5d2a473feb5f6b2ed4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 14:17:06 +0100 Subject: [PATCH 088/113] Fix tests --- src/libutil/tests/tests.cc | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 35a5d27bb..58df9c5ac 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -320,20 +320,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Float, emptyString) { - double n; - ASSERT_EQ(string2Float("", n), false); + ASSERT_EQ(string2Float(""), std::nullopt); } TEST(string2Float, trivialConversions) { - double n; - ASSERT_EQ(string2Float("1.0", n), true); - ASSERT_EQ(n, 1.0); + ASSERT_EQ(string2Float("1.0"), 1.0); - ASSERT_EQ(string2Float("0.0", n), true); - ASSERT_EQ(n, 0.0); + ASSERT_EQ(string2Float("0.0"), 0.0); - ASSERT_EQ(string2Float("-100.25", n), true); - ASSERT_EQ(n, (-100.25)); + ASSERT_EQ(string2Float("-100.25"), -100.25); } /* ---------------------------------------------------------------------------- @@ -341,20 +336,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Int, emptyString) { - double n; - ASSERT_EQ(string2Int("", n), false); + ASSERT_EQ(string2Int(""), std::nullopt); } TEST(string2Int, trivialConversions) { - double n; - ASSERT_EQ(string2Int("1", n), true); - ASSERT_EQ(n, 1); + ASSERT_EQ(string2Int("1"), 1); - ASSERT_EQ(string2Int("0", n), true); - ASSERT_EQ(n, 0); + ASSERT_EQ(string2Int("0"), 0); - ASSERT_EQ(string2Int("-100", n), true); - ASSERT_EQ(n, (-100)); + ASSERT_EQ(string2Int("-100"), -100); } /* ---------------------------------------------------------------------------- From 1db3f84baccc30ac38227c1f7edc3bfbc8e5ff5b Mon Sep 17 00:00:00 2001 From: Danila Date: Fri, 8 Jan 2021 16:12:21 -0800 Subject: [PATCH 089/113] Upcase "Boolean" in Flake attribute type error Co-authored-by: Eelco Dolstra --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9f1e4063f..61aeae543 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", attr.name, showType(*attr.value)); } } From fdcd62eec59485665b919c048874de05235b5971 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Jan 2021 23:20:02 +0100 Subject: [PATCH 090/113] Add 'nix store gc' command --- src/nix/store-gc.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ src/nix/store-gc.md | 21 +++++++++++++++++++++ tests/flakes.sh | 6 +++--- 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/nix/store-gc.cc create mode 100644 src/nix/store-gc.md diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..6e9607d03 --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "stop after freeing `n` bytes of disk space", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/tests/flakes.sh b/tests/flakes.sh index 5aec563ac..2b7bcdd68 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -276,18 +276,18 @@ git -C $flake3Dir commit -m 'Add lockfile' # Test whether registry caching works. nix registry list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp -nix-store --gc +nix store gc nix registry list --flake-registry file://$registry --refresh | grep -q flake3 mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache -nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache +nix store gc # get rid of copies in the store to ensure they get fetched to our git cache _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar mv $flake1Dir $flake1Dir.tmp mv $flake2Dir $flake2Dir.tmp -nix-store --gc +nix store gc _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh mv $flake1Dir.tmp $flake1Dir From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 12:36:39 +0100 Subject: [PATCH 091/113] nix store prefetch-tarball -> nix flake prefetch --- src/nix/flake-prefetch.md | 28 +++++++++++ src/nix/flake.cc | 40 ++++++++++++++++ src/nix/prefetch.cc | 77 +++++++++---------------------- src/nix/store-prefetch-tarball.md | 31 ------------- 4 files changed, 89 insertions(+), 87 deletions(-) create mode 100644 src/nix/flake-prefetch.md delete mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2b91faa64..b73b9cf4e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -960,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand } }; +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + struct CmdFlake : NixMultiCommand { CmdFlake() @@ -973,6 +1012,7 @@ struct CmdFlake : NixMultiCommand {"clone", []() { return make_ref(); }}, {"archive", []() { return make_ref(); }}, {"show", []() { return make_ref(); }}, + {"prefetch", []() { return make_ref(); }}, }) { } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 969299489..ce8c85ecf 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv) static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); -struct CmdStorePrefetch : StoreCommand, MixJSON +struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; - bool unpack; std::optional name; HashType hashType = htSHA256; std::optional expectedHash; - CmdStorePrefetch(bool unpack) - : unpack(unpack) + CmdStorePrefetchFile() { addFlag({ .longName = "name", @@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag({ .longName = "expected-hash", - .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .description = "expected hash of the file", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,14 +274,31 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + expectArg("url", &url); } Category category() override { return catUtility; } + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); @@ -299,54 +314,4 @@ struct CmdStorePrefetch : StoreCommand, MixJSON } }; -struct CmdStorePrefetchFile : CmdStorePrefetch -{ - CmdStorePrefetchFile() - : CmdStorePrefetch(false) - { - name = "source"; - - addFlag({ - .longName = "executable", - .description = "make the resulting file executable", - .handler = {&executable, true}, - }); - } - - std::string description() override - { - return "download a file into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-file.md" - ; - } -}; - static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); - -struct CmdStorePrefetchTarball : CmdStorePrefetch -{ - CmdStorePrefetchTarball() - : CmdStorePrefetch(true) - { - name = "source"; - } - - std::string description() override - { - return "download and unpack a tarball into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-tarball.md" - ; - } -}; - -static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md deleted file mode 100644 index 535d7e022..000000000 --- a/src/nix/store-prefetch-tarball.md +++ /dev/null @@ -1,31 +0,0 @@ -R""( - -# Examples - -* Download a tarball and unpack it: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz - Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' - to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash - 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). - ``` - -* Download a tarball and unpack it, unless it already exists in the - Nix store: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ - --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= - ``` - -# Description - -This command downloads a tarball or zip file from *url*, unpacks it, -and adds the unpacked tree to the Nix store. It prints out the -resulting store path and the NAR hash of that store path. - -The name component of the store path defaults to `source`, but this -can be overriden using `--name`. - -)"" From 77c9ceda4be8dd304b07f654d8c059a01d839108 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:42:24 +0100 Subject: [PATCH 092/113] Tweak --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 827adbd05..361c20cc9 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -226,7 +226,7 @@ control what gets deleted and in what order: or TiB units. The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` variables in the Nix +`keep-outputs` and `keep-derivations` settings in the Nix configuration file. By default, the collector prints the total number of freed bytes when it From 6254b1f5d298ff73127d7b0f0da48f142bdc753c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:46:17 +0100 Subject: [PATCH 093/113] Add 'nix store delete' command --- src/nix/store-delete.cc | 45 +++++++++++++++++++++++++++++++++++++++ src/nix/store-delete.md | 24 +++++++++++++++++++++ tests/multiple-outputs.sh | 2 +- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/nix/store-delete.cc create mode 100644 src/nix/store-delete.md diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..f3677763c --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,45 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "do not check whether the paths are reachable from a root", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 7a6ec181d..de573d4fa 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou # Delete one of the outputs and rebuild it. This will cause a hash # rewrite. -nix-store --delete $TEST_ROOT/result-second --ignore-liveness +nix store delete $TEST_ROOT/result-second --ignore-liveness nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result [ "$(cat $TEST_ROOT/result-second/file)" = "second" ] [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Tue, 12 Jan 2021 01:28:00 +0100 Subject: [PATCH 094/113] Don't let 'preferLocalBuild' override 'max-jobs=0' This resolves #3810 by changing the behavior of `max-jobs = 0`, so that specifying the option also avoids local building of derivations with the attribute `preferLocalBuild = true`. --- src/libstore/parsed-derivations.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 11 Jan 2021 22:05:32 -0600 Subject: [PATCH 095/113] Set kern.curproc_arch_affinity=0 to escape Rosetta By default, once you enter x86_64 Rosetta 2, macOS will try to run everything in x86_64. So an x86_64 Nix will still try to use x86_64 even when system = aarch64-darwin. To avoid this we can set kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0, we ignore this preference. This is based on how https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html works. Completely undocumented, but seems to work! Note, you can verify this works with this impure Nix expression: ``` { a = derivation { name = "a"; system = "aarch64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = arm64 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; b = derivation { name = "b"; system = "x86_64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; } ``` --- src/libstore/build/derivation-goal.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..a02ddb950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,6 +52,7 @@ #if __APPLE__ #include +#include #endif #include @@ -2869,6 +2870,10 @@ void DerivationGoal::runChild() throw SysError("failed to initialize builder"); if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } else if (drv->platform == "x86_64-darwin") { From 29007f8bc6ea42ae1f8311f00c0b5e14f04ec9e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 19:57:05 +0100 Subject: [PATCH 096/113] nix profile info -> nix profile list --- src/nix/{profile-info.md => profile-list.md} | 2 +- src/nix/profile.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/nix/{profile-info.md => profile-list.md} (98%) diff --git a/src/nix/profile-info.md b/src/nix/profile-list.md similarity index 98% rename from src/nix/profile-info.md rename to src/nix/profile-list.md index a0c04fc8c..5c29c0b02 100644 --- a/src/nix/profile-info.md +++ b/src/nix/profile-list.md @@ -5,7 +5,7 @@ R""( * Show what packages are installed in the default profile: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cdd34a20..ac60d336c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { @@ -346,7 +346,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro std::string doc() override { return - #include "profile-info.md" + #include "profile-list.md" ; } @@ -408,7 +408,7 @@ struct CmdProfile : NixMultiCommand {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, + {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, }) { } From 2f463e90ed077e066455a9ef6e024b18fd61c4de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 23:51:07 +0100 Subject: [PATCH 097/113] Add 'nix profile history' command Replaces 'nix-env --list-generations'. Similar to 'nix profile diff-closures' but shows only the changes in top-level packages. --- src/nix/command.hh | 2 + src/nix/profile-history.md | 26 +++++++++ src/nix/profile.cc | 114 +++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/nix/profile-history.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 6882db195..3aae57edd 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -261,6 +261,8 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); +std::string showVersions(const std::set & versions); + void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..d0fe40c82 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 -> 509: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 -> 510: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ac60d336c..ca95817d0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include "../nix-env/user-env.hh" #include "profiles.hh" +#include "names.hh" #include #include @@ -21,6 +22,13 @@ struct ProfileElementSource FlakeRef resolvedRef; std::string attrPath; // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } }; struct ProfileElement @@ -29,6 +37,29 @@ struct ProfileElement std::optional source; bool active = true; // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } }; struct ProfileManifest @@ -142,6 +173,46 @@ struct ProfileManifest return std::move(info.path); } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } }; struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile @@ -401,6 +472,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile } }; +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + if (prevGen) + std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); + else + std::cout << fmt("Version %d:\n", gen.number); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -410,6 +523,7 @@ struct CmdProfile : NixMultiCommand {"upgrade", []() { return make_ref(); }}, {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, + {"history", []() { return make_ref(); }}, }) { } From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 13 Jan 2021 10:13:51 +0100 Subject: [PATCH 098/113] Remove a redundant condition in DerivationGoal::tryLocalBuild() --- src/libstore/build/derivation-goal.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..415a55d37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 14:18:04 +0100 Subject: [PATCH 099/113] Convert option descriptions to Markdown --- src/libexpr/common-eval-args.cc | 10 +++++----- src/libmain/common-args.cc | 13 ++++++------- src/libmain/common-args.hh | 4 ++-- src/libmain/shared.cc | 20 ++++++++++---------- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 6 +++--- src/nix/bundle.cc | 6 +++--- src/nix/command.cc | 14 +++++++------- src/nix/copy.cc | 8 ++++---- src/nix/develop.cc | 16 ++++++++-------- src/nix/eval.cc | 6 +++--- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 19 +++++++++---------- src/nix/installables.cc | 26 +++++++++++++------------- src/nix/ls.cc | 6 +++--- src/nix/main.cc | 14 +++++++------- src/nix/path-info.cc | 8 ++++---- src/nix/prefetch.cc | 8 +++++--- src/nix/run.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 6 +++--- src/nix/store-delete.cc | 2 +- src/nix/store-gc.cc | 2 +- src/nix/upgrade-nix.cc | 4 ++-- src/nix/verify.cc | 8 ++++---- src/nix/why-depends.cc | 2 +- 26 files changed, 111 insertions(+), 111 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..ffe782454 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs() { addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up `<...>` file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); addFlag({ .longName = "impure", - .description = "allow access to mutable paths and repositories", + .description = "Allow access to mutable paths and repositories.", .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", - .description = "override a flake registry value", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3e4e475e5..bd5573e5d 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "verbose", .shortName = 'v', - .description = "increase verbosity level", + .description = "Increase the logging verbosity level.", .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", - .description = "decrease verbosity level", + .description = "Decrease the logging verbosity level.", .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", - .description = "enable debug output", + .description = "Set the logging verbosity level to 'debug'.", .handler = {[]() { verbosity = lvlDebug; }}, }); addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding `nix.conf`)", + .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { @@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "format of log output; `raw`, `internal-json`, `bar` " - "or `bar-with-logs`", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "max-jobs", .shortName = 'j', - .description = "maximum number of parallel builds", + .description = "The maximum number of parallel builds.", .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index a4de3dccf..47f341619 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -16,7 +16,7 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); } }; @@ -26,7 +26,7 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "produce JSON output", &json); + mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); } }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e797c2fb9..7e27e95c2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -186,27 +186,27 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-build-output", .shortName = 'Q', - .description = "do not show build output", + .description = "Do not show build output.", .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ .longName = "keep-failed", .shortName ='K', - .description = "keep temporary directories of failed builds", + .description = "Keep temporary directories of failed builds.", .handler = {&(bool&) settings.keepFailed, true}, }); addFlag({ .longName = "keep-going", .shortName ='k', - .description = "keep going after a build fails", + .description = "Keep going after a build fails.", .handler = {&(bool&) settings.keepGoing, true}, }); addFlag({ .longName = "fallback", - .description = "build from source if substitution fails", + .description = "Build from source if substitution fails.", .handler = {&(bool&) settings.tryFallback, true}, }); @@ -225,19 +225,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); }; - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores"); + intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); + intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "do not write to the Nix store", + mkFlag(0, "readonly-mode", "Do not write to the Nix store.", &settings.readOnlyMode); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", &gcWarning, false); addFlag({ .longName = "store", - .description = "URI of the Nix store to use", + .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, .handler = {&(std::string&) settings.storeUri}, }); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ea4bbbab9..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); diff --git a/src/nix/build.cc b/src/nix/build.cc index c2974d983..4cb8ade08 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); addFlag({ .longName = "rebuild", - .description = "rebuild an already built package and compare the result to the existing store paths", + .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, }); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5f558b01e..1789e4598 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand { addFlag({ .longName = "bundler", - .description = "use custom bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { @@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -90,7 +90,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/command.cc b/src/nix/command.cc index 596217775..ba58c7d6b 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) if (recursive) addFlag({ .longName = "no-recursive", - .description = "apply operation to specified paths only", + .description = "Apply operation to specified paths only.", .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', - .description = "apply operation to closure of the specified paths", + .description = "Apply operation to closure of the specified paths.", .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "apply operation to the entire store", &all); + mkFlag(0, "all", "Apply the operation to every store path.", &all); } void StorePathsCommand::run(ref store) @@ -133,7 +133,7 @@ MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "profile to update", + .description = "The profile to update.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "ignore-environment", .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", + .description = "Clear the entire environment (except those specified with `--keep`).", .handler = {&ignoreEnvironment, true}, }); addFlag({ .longName = "keep", .shortName = 'k', - .description = "keep specified environment variable", + .description = "Keep the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { keep.insert(s); }}, }); @@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "unset", .shortName = 'u', - .description = "unset specified environment variable", + .description = "Unset the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { unset.insert(s); }}, }); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2394eb46d..f15031a45 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index edd87f246..578258394 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile { addFlag({ .longName = "redirect", - .description = "redirect a store path to a mutable location", + .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); @@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed instead of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "phase", - .description = "phase to run (e.g. `build` or `configure`)", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", .labels = {"phase-name"}, .handler = {&phase}, }); addFlag({ .longName = "configure", - .description = "run the configure phase", + .description = "Run the `configure` phase.", .handler = {&phase, {"configure"}}, }); addFlag({ .longName = "build", - .description = "run the build phase", + .description = "Run the `build` phase.", .handler = {&phase, {"build"}}, }); addFlag({ .longName = "check", - .description = "run the check phase", + .description = "Run the `check` phase.", .handler = {&phase, {"check"}}, }); addFlag({ .longName = "install", - .description = "run the install phase", + .description = "Run the `install` phase.", .handler = {&phase, {"install"}}, }); addFlag({ .longName = "installcheck", - .description = "run the installcheck phase", + .description = "Run the `installcheck` phase.", .handler = {&phase, {"installCheck"}}, }); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 321df7495..b5049ac65 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); addFlag({ .longName = "apply", - .description = "apply a function to each argument", + .description = "Apply the function *expr* to each argument.", .labels = {"expr"}, .handler = {&apply}, }); addFlag({ .longName = "write-to", - .description = "write a string or attrset of strings to 'path'", + .description = "Write a string or attrset of strings to *path*.", .labels = {"path"}, .handler = {&writeTo}, }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b73b9cf4e..4cd7d77a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -222,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand { addFlag({ .longName = "no-build", - .description = "do not build checks", + .description = "Do not build checks.", .handler = {&build, false} }); } @@ -573,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand addFlag({ .longName = "template", .shortName = 't', - .description = "the template to use", + .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, .completer = {[&](size_t, std::string_view prefix) { @@ -717,7 +717,7 @@ struct CmdFlakeClone : FlakeCommand addFlag({ .longName = "dest", .shortName = 'f', - .description = "destination path", + .description = "Clone the flake to path *dest*.", .labels = {"path"}, .handler = {&destDir} }); @@ -807,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand { addFlag({ .longName = "legacy", - .description = "show the contents of the 'legacyPackages' output", + .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6fd791f41..79d506ace 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,15 +19,15 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); + mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); + mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); + mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") - .description("compute hash modulo specified string") + .description("Compute the hash modulo specified the string.") .labels({"modulus"}) .dest(&modulus); #endif @@ -40,15 +40,14 @@ struct CmdHashBase : Command std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; - break; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } void run() override diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3506c3fcc..50e3b29c4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions() { addFlag({ .longName = "recreate-lock-file", - .description = "recreate lock file from scratch", + .description = "Recreate the flake's lock file from scratch.", .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", - .description = "do not allow any updates to the lock file", + .description = "Do not allow any updates to the flake's lock file.", .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", - .description = "do not write the newly generated lock file", + .description = "Do not write the flake's newly generated lock file.", .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", - .description = "don't use flake registries", + .description = "Don't allow lookups in the flake registries.", .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", - .description = "commit changes to the lock file", + .description = "Commit changes to the flake's lock file.", .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", - .description = "update a specific flake input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", - .description = "use the inputs of the specified flake as registry entries", + .description = "Use the inputs of the specified flake as registry entries.", .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "evaluate *file* rather than the default", + .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", .labels = {"file"}, .handler = {&file}, .completer = completePath }); addFlag({ - .longName ="expr", - .description = "evaluate attributes from *expr*", + .longName = "expr", + .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", .labels = {"expr"}, .handler = {&expr} }); addFlag({ - .longName ="derivation", - .description = "operate on the store derivation rather than its outputs", + .longName = "derivation", + .description = "Operate on the store derivation rather than its outputs.", .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d48287f27..c0b1ecb32 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); + mkFlag('l', "long", "Show detailed file information.", &verbose); + mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); } void listText(ref accessor) diff --git a/src/nix/main.cc b/src/nix/main.cc index b2406fafe..803453dd5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -69,15 +69,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", + .description = "Show usage information.", .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ .longName = "help-config", - .description = "show configuration options", + .description = "Show configuration settings.", .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; + std::cout << "The following configuration settings are available:\n\n"; Table2 tbl; std::map settings; globalConfig.getSettings(settings); @@ -91,25 +91,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", + .description = "Show version information.", .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 30b6a50f8..0fa88f1bf 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); + mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); + mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); + mkFlag(0, "sigs", "Show signatures.", &showSigs); } std::string description() override diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce8c85ecf..a831dcd15 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { addFlag({ .longName = "name", - .description = "store path name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, .handler = {&name} }); addFlag({ .longName = "expected-hash", - .description = "expected hash of the file", + .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON addFlag({ .longName = "executable", - .description = "make the resulting file executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", .handler = {&executable, true}, }); diff --git a/src/nix/run.cc b/src/nix/run.cc index 1340dd46f..ec9388234 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 13f2c8e69..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 14e2c9761..4b6ead6c7 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -101,7 +101,7 @@ struct CmdSignPaths : StorePathsCommand addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath @@ -150,7 +150,7 @@ struct CmdKeyGenerateSecret : Command { addFlag({ .longName = "key-name", - .description = "identifier of the key (e.g. `cache.example.org-1`)", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", .labels = {"name"}, .handler = {&keyName}, }); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index f3677763c..9c8fef191 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -13,7 +13,7 @@ struct CmdStoreDelete : StorePathsCommand { addFlag({ .longName = "ignore-liveness", - .description = "do not check whether the paths are reachable from a root", + .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true} }); } diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 6e9607d03..a2d74066e 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -13,7 +13,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun { addFlag({ .longName = "max", - .description = "stop after freeing `n` bytes of disk space", + .description = "Stop after freeing *n* bytes of disk space.", .labels = {"n"}, .handler = {&options.maxFreed} }); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 79be31e73..299ea40aa 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 620109aac..b2963cf74 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,13 +18,13 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); + mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); @@ -32,7 +32,7 @@ struct CmdVerify : StorePathsCommand addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "require that each path has at least N valid signatures", + .description = "Require that each path has at least *n* valid signatures.", .labels = {"n"}, .handler = {&sigsNeeded} }); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 297b638cc..7a4ca5172 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:27:39 +0100 Subject: [PATCH 100/113] Add 'nix store repair' command --- src/libstore/local-store.hh | 4 +--- src/libstore/store-api.hh | 5 +++++ src/nix-store/nix-store.cc | 2 +- src/nix/store-delete.cc | 1 - src/nix/store-repair.cc | 27 +++++++++++++++++++++++++++ src/nix/store-repair.md | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/nix/store-repair.cc create mode 100644 src/nix/store-repair.md diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d29c5960..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..d1b83933a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -604,6 +604,11 @@ public: virtual ref getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e43788bc3..b97f684a4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -757,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); + store->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 9c8fef191..10245978e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,6 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector storePaths) override { - for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" From d33eca8539d2e66759f7b52fa7b0db4a6a1ba673 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:31:18 +0100 Subject: [PATCH 101/113] Rename 'nix store sign-paths' to 'nix store sign' --- doc/manual/src/advanced-topics/post-build-hook.md | 4 ++-- src/nix/key-generate-secret.md | 2 +- src/nix/main.cc | 2 +- src/nix/sigs.cc | 6 +++--- tests/signing.sh | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index bbdabed41..fcb52d878 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -53,7 +53,7 @@ set -f # disable globbing export IFS=' ' echo "Signing paths" $OUT_PATHS -nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS exec nix copy --to 's3://example-nix-cache' $OUT_PATHS ``` @@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix -> sign-paths`. Nix guarantees the paths will not contain any spaces, +> store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md index 6ff1e1c9b..4938f637c 100644 --- a/src/nix/key-generate-secret.md +++ b/src/nix/key-generate-secret.md @@ -12,7 +12,7 @@ R""( ```console # nix build nixpkgs#hello - # nix store sign-paths --key-file ./secret-key --recursive ./result + # nix store sign --key-file ./secret-key --recursive ./result ``` Finally, we can verify the store paths using the corresponding diff --git a/src/nix/main.cc b/src/nix/main.cc index 803453dd5..398526020 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -129,7 +129,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"make-content-addressable", {"store", "make-content-addressable"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign-paths"}}, + {"sign-paths", {"store", "sign"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 4b6ead6c7..3445182f2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,11 +92,11 @@ struct CmdCopySigs : StorePathsCommand static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", @@ -140,7 +140,7 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); +static auto rCmdSign = registerCommand2({"store", "sign"}); struct CmdKeyGenerateSecret : Command { diff --git a/tests/signing.sh b/tests/signing.sh index bd6280cc6..6aafbeb91 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -47,8 +47,8 @@ expect 2 nix store verify -r $outPath2 --sigs-needed 1 expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix store sign-paths". -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign". +nix store sign --key-file $TEST_ROOT/sk1 $outPath2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 @@ -63,7 +63,7 @@ nix store verify $outPathCA nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store sign --key-file $TEST_ROOT/sk1 $outPathCA nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] From 28ef6ebf914792f9e543e9778248f06e716a859d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:51:27 +0100 Subject: [PATCH 102/113] Typo --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index fc6195cf0..9227369b8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -318,7 +318,7 @@ static int main_nix_daemon(int argc, char * * argv) FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limitting access to thoses is explicitly + standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } From 7a472a76d4dcbbd0eb7832c0bdcb120d32881e8b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jan 2021 00:05:04 +0100 Subject: [PATCH 103/113] Add 'nix daemon' command --- src/nix/command.hh | 2 + .../nix-daemon.cc => nix/daemon.cc} | 114 +++++++++++------- src/nix/daemon.md | 21 ++++ src/nix/main.cc | 3 + tests/common.sh.in | 2 +- 5 files changed, 98 insertions(+), 44 deletions(-) rename src/{nix-daemon/nix-daemon.cc => nix/daemon.cc} (77%) create mode 100644 src/nix/daemon.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 3aae57edd..f325cd906 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -13,6 +13,8 @@ namespace nix { extern std::string programPath; +extern char * * savedArgv; + class EvalState; struct Pos; class Store; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc similarity index 77% rename from src/nix-daemon/nix-daemon.cc rename to src/nix/daemon.cc index 9227369b8..204d4ce6b 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,4 @@ +#include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" @@ -150,7 +151,7 @@ static ref openUncachedStore() } -static void daemonLoop(char * * argv) +static void daemonLoop() { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -232,9 +233,9 @@ static void daemonLoop(char * * argv) setSigChldAction(false); // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && argv[1]) { + if (peer.pidKnown && savedArgv[1]) { string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); } // Handle the connection. @@ -264,6 +265,48 @@ static void daemonLoop(char * * argv) } } +static void runDaemon(bool stdio) +{ + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + } + } else + daemonLoop(); +} static int main_nix_daemon(int argc, char * * argv) { @@ -285,49 +328,34 @@ static int main_nix_daemon(int argc, char * * argv) initPlugins(); - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } - } else { - daemonLoop(argv); - } + runDaemon(stdio); return 0; } } static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index 398526020..418396280 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -52,6 +52,7 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { @@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand("help"); void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { diff --git a/tests/common.sh.in b/tests/common.sh.in index 5489c0c44..e3bcab507 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -73,7 +73,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix-daemon & + nix daemon & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 From 86a2ceeb986609488be1c6794a8e416df3b90c7b Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Wed, 13 Jan 2021 16:56:23 -0800 Subject: [PATCH 104/113] Fix gcc10 build --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 61aeae543..0786fef3d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -128,7 +128,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, Explicit { attr.value->boolean }); break; case nInt: - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, (long unsigned int)attr.value->integer); break; default: throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", From 1e13c79a9165e99be9fccfec8e442d14bb66aef0 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 16 Jan 2021 19:11:10 +0000 Subject: [PATCH 105/113] Document expected output of 'nix store ping'. While interpreting the output is fairly intuitive it would be better to explicitly specify what a good invocation looks like. That this isn't completely obvious (or at least causes folks to second-guess themselves) can be seen in a couple user threads: - https://discourse.nixos.org/t/nixos-cache-fetching-issue/3575/11 - https://discourse.nixos.org/t/newbie-question-cant-get-trivial-example-of-nixops-to-work-on-my-mac/1125/8 --- src/nix/ping-store.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 322093091..79b108d9c 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,4 +27,6 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. +When the command succeeds a zero exit code is returned with no output. + )"" From 1acbb61696c118712417bcd1c59021cc84650e16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 17 Jan 2021 19:49:28 +0100 Subject: [PATCH 106/113] Tweak --- src/nix/ping-store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 79b108d9c..8c846791b 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,6 +27,7 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. -When the command succeeds a zero exit code is returned with no output. +If the command succeeds, Nix returns a exit code of 0 and does not +print any output. )"" From 9432c170e736a6b506d9b35ced5eccff6422ec50 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 17:12:58 +0100 Subject: [PATCH 107/113] Fix the drv output map for non ca derivations With the `ca-derivation` experimental features, non-ca derivations used to have their output paths returned as unknown as long as they weren't built (because of a mistake in the code that systematically erased the previous value) --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c61f34275..ab78f1435 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -914,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); else - outputs.insert_or_assign(outputName, std::nullopt); + outputs.insert({outputName, std::nullopt}); } return outputs; From 11b63740e377202e237b7bc74806b82a1eb8ce11 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 21:26:29 +0100 Subject: [PATCH 108/113] Fix content-addressed flake outputs Prevent some `nix flake` commands to crash by trying to parse a placeholder output as a store path --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 50e3b29c4..34ee238bf 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -501,7 +501,7 @@ std::tuple InstallableF auto drvInfo = DerivationInfo{ std::move(drvPath), - state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), attr->getAttr(state->sOutputName)->getString() }; From ea756b3654931f23839aee9f461a8c891c6ffe43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 14:38:31 +0100 Subject: [PATCH 109/113] --refresh: Imply setting .narinfo disk cache TTL to 0 --- src/libstore/nar-info-disk-cache.cc | 6 ++++-- src/nix/main.cc | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..1d8d2d57e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -109,8 +109,10 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) + // Use a minimum TTL to prevent --refresh from + // nuking the entire disk cache. + (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U)) + (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); diff --git a/src/nix/main.cc b/src/nix/main.cc index 418396280..80422bd24 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -330,8 +330,11 @@ void mainWrapped(int argc, char * * argv) fileTransferSettings.connectTimeout = 1; } - if (args.refresh) + if (args.refresh) { settings.tarballTtl = 0; + settings.ttlNegativeNarInfoCache = 0; + settings.ttlPositiveNarInfoCache = 0; + } args.command->second->prepare(); args.command->second->run(); From 555940f0659e95de7f890ede48e2faba096b3d6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 22:50:39 +0100 Subject: [PATCH 110/113] Use enumerate() --- src/nix/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 4cb8ade08..45f63bb7e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (size_t i = 0; i < buildables.size(); ++i) + for (const auto & [i, buildable] : enumerate(buildables)) { std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; @@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildables[i]); + }, buildable); + } updateProfile(buildables); From bc90252cec9af05b897cf209012d44a9b20ea251 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 23:08:58 +0100 Subject: [PATCH 111/113] nix profile install: Support installing non-flakes Fixes #4458. --- src/nix/profile.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ca95817d0..765d6866e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -252,8 +252,28 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME manifest.elements.emplace_back(std::move(element)); - } else - throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what()); + } else { + auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + + for (auto & buildable : buildables) { + ProfileElement element; + + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path, {}}); + element.storePaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + pathsToBuild.push_back({bfd.drvPath, {output.first}}); + element.storePaths.insert(output.second); + } + }, + }, buildable); + + manifest.elements.emplace_back(std::move(element)); + } + } } store->buildPaths(pathsToBuild); From 259100332f96250d6615d5839f6a77798c77aefb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 10:29:51 +0100 Subject: [PATCH 112/113] Fix clang build --- src/nix/build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 45f63bb7e..724ce9d79 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (const auto & [i, buildable] : enumerate(buildables)) { + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; From b7bfc7ee52dd425e0156f369eb4c05a62358f912 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 14:54:28 +0100 Subject: [PATCH 113/113] Add FIXME --- src/libutil/error.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7dc3f1d3..2a67a730a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -61,6 +61,8 @@ std::optional getCodeLines(const ErrPos & errPos) if (errPos.origin == foFile) { LinesOfCode loc; try { + // FIXME: when running as the daemon, make sure we don't + // open a file to which the client doesn't have access. AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) return {};