From bd0192d0bbd19d5d2b6ac89e2b71264e396bf08d Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Tue, 31 May 2022 11:51:17 -0700 Subject: [PATCH 01/37] flake: update to 22.05 The static build works now :) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 31c1910df..01e4f506a 100644 --- a/flake.lock +++ b/flake.lock @@ -18,16 +18,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1645296114, - "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=", + "lastModified": 1653988320, + "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1", + "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-21.05-small", + "ref": "nixos-22.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a69969cfa..9a1442e52 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; From dbf0d3a150ef01b5451c1a04e6a6bcd67a3e4a86 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Tue, 31 May 2022 12:14:34 -0700 Subject: [PATCH 02/37] tests/nss-preload: move nix-fetch binding --- tests/nss-preload.nix | 70 ++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/tests/nss-preload.nix b/tests/nss-preload.nix index 2610d2b30..64b655ba2 100644 --- a/tests/nss-preload.nix +++ b/tests/nss-preload.nix @@ -5,6 +5,42 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; }; +let + nix-fetch = pkgs.writeText "fetch.nix" '' + derivation { + # This derivation is an copy from what is available over at + # nix.git:corepkgs/fetchurl.nix + builder = "builtin:fetchurl"; + + # We're going to fetch data from the http_dns instance created before + # we expect the content to be the same as the content available there. + # ``` + # $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1) + # 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59 + # ``` + outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59"; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + + name = "example.com"; + url = "http://example.com"; + + unpack = false; + executable = false; + + system = "builtin"; + + preferLocalBuild = true; + + impureEnvVars = [ + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + urls = [ "http://example.com" ]; + } + ''; +in + makeTest ( rec { @@ -68,40 +104,6 @@ rec { }; }; - nix-fetch = pkgs.writeText "fetch.nix" '' - derivation { - # This derivation is an copy from what is available over at - # nix.git:corepkgs/fetchurl.nix - builder = "builtin:fetchurl"; - - # We're going to fetch data from the http_dns instance created before - # we expect the content to be the same as the content available there. - # ``` - # $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1) - # 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59 - # ``` - outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59"; - outputHashAlgo = "sha256"; - outputHashMode = "flat"; - - name = "example.com"; - url = "http://example.com"; - - unpack = false; - executable = false; - - system = "builtin"; - - preferLocalBuild = true; - - impureEnvVars = [ - "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" - ]; - - urls = [ "http://example.com" ]; - } - ''; - testScript = { nodes, ... }: '' http_dns.wait_for_unit("nginx") http_dns.wait_for_open_port(80) From 159b5815b527f466578a2d28fbf832617cc45b88 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jan 2022 18:03:24 +0100 Subject: [PATCH 03/37] repl: `--option pure-eval true` actually enables pure eval mode To quote Eelco in #5867: > Unfortunately we can't do > > evalSettings.pureEval.setDefault(false); > > because then we have to do the same in main.cc (where > pureEval is set to true), and that would allow pure-eval > to be disabled globally from nix.conf. Instead, a command should specify that it should be impure by default. Then, `evalSettings.pureEval` will be set to `false;` unless it's overridden by e.g. a CLI flag. In that case it's IMHO OK to be (theoretically) able to override `pure-eval` via `nix.conf` because it doesn't have an effect on commands where `forceImpureByDefault` returns `false` (i.e. everything where pure eval actually matters). Closes #5867 --- src/libcmd/repl.cc | 7 +++++-- src/libutil/args.hh | 2 ++ src/nix/main.cc | 3 +++ tests/repl.sh | 5 +++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 458e824c5..3c89a8ea3 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1039,6 +1039,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs }); } + bool forceImpureByDefault() override + { + return true; + } + std::string description() override { return "start an interactive environment for evaluating Nix expressions"; @@ -1053,8 +1058,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref store) override { - evalSettings.pureEval = false; - auto evalState = make_ref(searchPath, store); auto repl = std::make_unique(evalState); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index fdd036f9a..07c017719 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -25,6 +25,8 @@ public: /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } + virtual bool forceImpureByDefault() { return false; } + /* Return documentation about this command, in Markdown format. */ virtual std::string doc() { return ""; } diff --git a/src/nix/main.cc b/src/nix/main.cc index dadb54306..f398e3118 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -380,6 +380,9 @@ void mainWrapped(int argc, char * * argv) settings.ttlPositiveNarInfoCache = 0; } + if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { + evalSettings.pureEval = false; + } args.command->second->prepare(); args.command->second->run(); } diff --git a/tests/repl.sh b/tests/repl.sh index b6937b9e9..9e6a59f18 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -42,6 +42,11 @@ testRepl () { echo "$replOutput" echo "$replOutput" | grep -qs "while evaluating the file" \ || fail "nix repl --show-trace doesn't show the trace" + + nix repl "${nixArgs[@]}" --option pure-eval true 2>&1 <<< "builtins.currentSystem" \ + | grep "attribute 'currentSystem' missing" + nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ + | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" } # Simple test, try building a drv From a9358a6097e0ec0491d4eb83c556c783128a2cb0 Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 1 Jun 2022 14:58:04 +0200 Subject: [PATCH 04/37] schema.sql: add comment about hash being in base16 --- src/libstore/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 09c71a2b8..d65e5335e 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,7 +1,7 @@ create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, - hash text not null, + hash text not null, -- base16 representation registrationTime integer not null, deriver text, narSize integer, From 505d6ee5e21654747c52e0877bf1d5982e8e9d31 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 1 Jun 2022 09:41:00 -0500 Subject: [PATCH 05/37] darwin-install: work around existing vim swapfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User on Matrix reported install problems which presented as "vifs:editing error" which we traced back to vim griping about an existing swap file. When opened interactively, it did this: E325: ATTENTION Found a swap file by the name "/etc/.fstab.swp" owned by: root dated: Sön Apr 24 16:54:10 2022 file name: /private/etc/fstab modified: YES user name: root host name: MBP.local process ID: 1698 While opening file "/etc/fstab" dated: Sön Apr 24 16:56:27 2022 NEWER than swap file! ... --- scripts/create-darwin-volume.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index aee7ff4bf..103e1e391 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -442,8 +442,9 @@ add_nix_vol_fstab_line() { local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - # wrap `ex` to work around a problem with vim plugins breaking exit codes - # (see github.com/NixOS/nix/issues/5468) + # wrap `ex` to work around problems w/ vim features breaking exit codes + # - plugins (see github.com/NixOS/nix/issues/5468): -u NONE + # - swap file: -n # # the first draft used `--noplugin`, but github.com/NixOS/nix/issues/6462 # suggests we need the less-semantic `-u NONE` @@ -456,7 +457,7 @@ add_nix_vol_fstab_line() { # minver 10.12.6 seems to have released with vim 7.4 cat > "$SCRATCH/ex_cleanroom_wrapper" <&2 # technically /etc/synthetic.d/nix is supported in Big Sur+ # but handling both takes even more code... - # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier + # See earlier note; `-u NONE` disables vim plugins/rc, `-n` skips swapfile _sudo "to add Nix to /etc/synthetic.conf" \ - /usr/bin/ex -u NONE /etc/synthetic.conf <&2 - # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier - _sudo "to install the Nix volume mounter" /usr/bin/ex -u NONE "$NIX_VOLUME_MOUNTD_DEST" < Date: Wed, 1 Jun 2022 16:33:03 +0200 Subject: [PATCH 06/37] Explain exactly what nix-upgrade nix does --- src/nix/upgrade-nix.cc | 2 +- src/nix/upgrade-nix.md | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17a5a77ee..2d2453395 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -34,7 +34,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the latest stable version"; + return "upgrade Nix to the stable version declared in Nixpkgs"; } std::string doc() override diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 4d27daad9..084c80ba2 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -2,7 +2,7 @@ R""( # Examples -* Upgrade Nix to the latest stable version: +* Upgrade Nix to the stable version declared in Nixpkgs: ```console # nix upgrade-nix @@ -16,8 +16,11 @@ R""( # Description -This command upgrades Nix to the latest version. By default, it -locates the directory containing the `nix` binary in the `$PATH` +This command upgrades Nix to the stable version declared in Nixpkgs. +This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +and updated manually. It may not always be the latest tagged release. + +By default, it locates the directory containing the `nix` binary in the `$PATH` environment variable. If that directory is a Nix profile, it will upgrade the `nix` package in that profile to the latest stable binary release. From 2868acb4a54c9f09ee94289e950da31aa1b5d541 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Jun 2022 17:01:28 +0200 Subject: [PATCH 07/37] tests/flakes.sh: Fix some ignored breakage --- tests/flakes.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 9a1f0ab6a..36bffcf3b 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -32,7 +32,7 @@ for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeD rm -rf $repo $repo.tmp mkdir -p $repo - # Give one repo a non-master initial branch. + # Give one repo a non-main initial branch. extraArgs= if [[ $repo == $flake2Dir ]]; then extraArgs="--initial-branch=main" @@ -173,11 +173,11 @@ nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file [[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff master) ]] +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] # Rerunning the build should not change the lockfile. nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff master) ]] +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] # Building with a lockfile should not require a fetch of the registry. nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh @@ -186,7 +186,7 @@ nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh # Updating the flake should not change the lockfile. nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff master) ]] +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] # Now we should be able to build the flake in pure mode. nix build -o $TEST_ROOT/result flake2#bar @@ -221,7 +221,7 @@ nix build -o $TEST_ROOT/result $flake3Dir#"sth sth" nix build -o $TEST_ROOT/result $flake3Dir#"sth%20sth" # Check whether it saved the lockfile -(! [[ -z $(git -C $flake3Dir diff master) ]]) +[[ -n $(git -C $flake3Dir diff master) ]] git -C $flake3Dir add flake.lock @@ -321,10 +321,10 @@ nix build -o $TEST_ROOT/result flake4#xyzzy # Test 'nix flake update' and --override-flake. nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master) ]] +[[ -z $(git -C $flake3Dir diff master || echo failed) ]] nix flake update $flake3Dir --override-flake flake2 nixpkgs -[[ ! -z $(git -C $flake3Dir diff master) ]] +[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C $flake3Dir checkout -b removeXyzzy From 81a486c607405027914d1f445bb570f19a4977b7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Jun 2022 16:55:28 +0200 Subject: [PATCH 08/37] Shut up clang warnings --- src/libexpr/eval.hh | 4 ++-- src/libexpr/nixexpr.hh | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7b8732169..4eaa3c9b0 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -150,7 +150,7 @@ public: if (debugRepl) runDebugRepl(&error, env, expr); - throw error; + throw std::move(error); } template @@ -165,7 +165,7 @@ public: runDebugRepl(&e, last.env, last.expr); } - throw e; + throw std::move(e); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8813c61a9..5eb022770 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -150,16 +150,16 @@ struct Expr }; #define COMMON_METHODS \ - void show(const SymbolTable & symbols, std::ostream & str) const; \ - void eval(EvalState & state, Env & env, Value & v); \ - void bindVars(EvalState & es, const std::shared_ptr & env); + void show(const SymbolTable & symbols, std::ostream & str) const override; \ + void eval(EvalState & state, Env & env, Value & v) override; \ + void bindVars(EvalState & es, const std::shared_ptr & env) override; struct ExprInt : Expr { NixInt n; Value v; ExprInt(NixInt n) : n(n) { v.mkInt(n); }; - Value * maybeThunk(EvalState & state, Env & env); + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -168,7 +168,7 @@ struct ExprFloat : Expr NixFloat nf; Value v; ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; - Value * maybeThunk(EvalState & state, Env & env); + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -177,7 +177,7 @@ struct ExprString : Expr std::string s; Value v; ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; - Value * maybeThunk(EvalState & state, Env & env); + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -186,7 +186,7 @@ struct ExprPath : Expr std::string s; Value v; ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; - Value * maybeThunk(EvalState & state, Env & env); + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -213,7 +213,7 @@ struct ExprVar : Expr ExprVar(Symbol name) : name(name) { }; ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; - Value * maybeThunk(EvalState & state, Env & env); + Value * maybeThunk(EvalState & state, Env & env) override; PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -326,7 +326,7 @@ struct ExprLambda : Expr : pos(pos), formals(formals), body(body) { } - void setName(Symbol name); + void setName(Symbol name) override; std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } @@ -395,15 +395,15 @@ struct ExprOpNot : Expr Expr * e1, * e2; \ name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ name(const PosIdx & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ - void show(const SymbolTable & symbols, std::ostream & str) const \ + void show(const SymbolTable & symbols, std::ostream & str) const override \ { \ str << "("; e1->show(symbols, str); str << " " s " "; e2->show(symbols, str); str << ")"; \ } \ - void bindVars(EvalState & es, const std::shared_ptr & env) \ + void bindVars(EvalState & es, const std::shared_ptr & env) override \ { \ e1->bindVars(es, env); e2->bindVars(es, env); \ } \ - void eval(EvalState & state, Env & env, Value & v); \ + void eval(EvalState & state, Env & env, Value & v) override; \ PosIdx getPos() const override { return pos; } \ }; From 24b3a500a747cb984001c3ca4384525583966692 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 May 2022 15:32:46 +0200 Subject: [PATCH 09/37] Typo --- tests/fetchTree-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh index 1c0ce39ce..f0c530466 100644 --- a/tests/fetchTree-file.sh +++ b/tests/fetchTree-file.sh @@ -58,7 +58,7 @@ EOF nix eval --file - < Date: Mon, 9 May 2022 14:28:27 +0200 Subject: [PATCH 10/37] Add operator for concatenating strings and string_views --- src/libutil/util.hh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 09ccfa591..16fa6c54c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -700,4 +700,19 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); +/* Provide an addition operator between strings and string_views + inexplicably omitted from the standard library. */ +inline std::string operator + (const std::string & s1, std::string_view s2) +{ + auto s = s1; + s.append(s2); + return s; +} + +inline std::string operator + (std::string && s, std::string_view s2) +{ + s.append(s2); + return s; +} + } From 28e08822a360d396260cec42d92704a3663e6aa9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Jun 2022 16:48:53 +0200 Subject: [PATCH 11/37] Avoid unnecessary string copy --- src/libutil/util.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 16fa6c54c..90418b04d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -712,7 +712,7 @@ inline std::string operator + (const std::string & s1, std::string_view s2) inline std::string operator + (std::string && s, std::string_view s2) { s.append(s2); - return s; + return std::move(s); } } From d137ceccefe08250106dcede1f30c270b0f9cf19 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 18:44:37 +0200 Subject: [PATCH 12/37] Fix incorrect comment in `hiliteMatches` --- src/libutil/hilite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/hilite.cc b/src/libutil/hilite.cc index a5991ca39..e5088230d 100644 --- a/src/libutil/hilite.cc +++ b/src/libutil/hilite.cc @@ -8,9 +8,9 @@ std::string hiliteMatches( std::string_view prefix, std::string_view postfix) { - // Avoid copy on zero matches + // Avoid extra work on zero matches if (matches.size() == 0) - return (std::string) s; + return std::string(s); std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { return a.position() < b.position(); From 0cd560c95dd981bde84c93379f6af677d31a2d0b Mon Sep 17 00:00:00 2001 From: Jonpez2 Date: Mon, 6 Jun 2022 16:56:42 +0100 Subject: [PATCH 13/37] Add security.csm to ignored-acls The security.csm ACL is, as far as I know, never reasonable to remove, so let's add it to the ignore-list in the vanilla nix image. This makes this image usable on GKE. --- docker.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/docker.nix b/docker.nix index 0cd64856f..a236d61d3 100644 --- a/docker.nix +++ b/docker.nix @@ -125,6 +125,7 @@ let sandbox = "false"; build-users-group = "nixbld"; trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + ignored-acls = security.csm; }; nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; From 5a9d83aa59df15ea9e289157518f64818036e020 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jun 2022 13:56:57 +0200 Subject: [PATCH 14/37] Disable cross builds on platforms other than x86_64-linux Needed because evaluation was broken on x86_64-darwin. --- flake.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9a1442e52..e936b5249 100644 --- a/flake.nix +++ b/flake.nix @@ -610,7 +610,9 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map (crossSystem: { + } + + // builtins.listToAttrs (map (crossSystem: { name = "nix-${crossSystem}"; value = let nixpkgsCross = import nixpkgs { @@ -649,7 +651,9 @@ doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; }; - }) crossSystems)) // (builtins.listToAttrs (map (stdenvName: + }) (if system == "x86_64-linux" then crossSystems else []))) + + // (builtins.listToAttrs (map (stdenvName: nixpkgsFor.${system}.lib.nameValuePair "nix-${stdenvName}" nixpkgsFor.${system}."${stdenvName}Packages".nix From 0f8754cd30ecbcfa49304d74853c3c0bbdd65d45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jun 2022 13:59:36 +0200 Subject: [PATCH 15/37] Fix 22.05 eval warnings --- flake.nix | 9 ++++----- tests/github-flakes.nix | 2 +- tests/nix-copy-closure.nix | 4 ++-- tests/remote-builds.nix | 2 +- tests/setuid.nix | 4 ++-- tests/sourcehut-flakes.nix | 2 +- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index e936b5249..217c83727 100644 --- a/flake.nix +++ b/flake.nix @@ -88,7 +88,6 @@ "LDFLAGS=-fuse-ld=gold" ]; - nativeBuildDeps = [ buildPackages.bison @@ -370,10 +369,10 @@ ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - configureFlags = '' - --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - ''; + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" + ]; enableParallelBuilding = true; diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index 7ac397d81..ddae6a21c 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -103,7 +103,7 @@ makeTest ( { config, lib, pkgs, nodes, ... }: { virtualisation.writableStore = true; virtualisation.diskSize = 2048; - virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; virtualisation.memorySize = 4096; nix.binaryCaches = lib.mkForce [ ]; nix.extraOptions = "experimental-features = nix-command flakes"; diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 1b63a3fca..ba8b2cfc9 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -14,7 +14,7 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk { client = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgA pkgD.drvPath ]; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; nix.binaryCaches = lib.mkForce [ ]; }; @@ -22,7 +22,7 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgB pkgC ]; + virtualisation.additionalPaths = [ pkgB pkgC ]; }; }; diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index b9e7352c0..7b2e6f708 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -61,7 +61,7 @@ in } ]; virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; nix.binaryCaches = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; }; diff --git a/tests/setuid.nix b/tests/setuid.nix index 35eb304ed..a83b1fc3a 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -10,12 +10,12 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { makeTest { name = "setuid"; - machine = + nodes.machine = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; nix.binaryCaches = lib.mkForce [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; - virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; }; testScript = { nodes }: '' diff --git a/tests/sourcehut-flakes.nix b/tests/sourcehut-flakes.nix index 6a1930904..aadab9bb5 100644 --- a/tests/sourcehut-flakes.nix +++ b/tests/sourcehut-flakes.nix @@ -106,7 +106,7 @@ makeTest ( { virtualisation.writableStore = true; virtualisation.diskSize = 2048; - virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; virtualisation.memorySize = 4096; nix.binaryCaches = lib.mkForce [ ]; nix.extraOptions = '' From faf80fa9200f0f7f0dfa3f510d7c8eb0975102e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Feb 2022 15:05:07 +0100 Subject: [PATCH 16/37] Convert to new flake style https://github.com/NixOS/nix/issues/5532 --- flake.nix | 72 +++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/flake.nix b/flake.nix index a69969cfa..f7f2379e7 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ ) ); - forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); + forAllStdenvs = f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = @@ -405,7 +405,7 @@ # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. - overlay = overlayFor (p: p.stdenv); + overlays.default = overlayFor (p: p.stdenv); hydraJobs = { @@ -430,7 +430,7 @@ value = let nixpkgsCross = import nixpkgs { inherit system crossSystem; - overlays = [ self.overlay ]; + overlays = [ self.overlays.default ]; }; in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; }) crossSystems)); @@ -476,31 +476,31 @@ tests.remoteBuilds = import ./tests/remote-builds.nix { system = "x86_64-linux"; inherit nixpkgs; - inherit (self) overlay; + overlay = self.overlays.default; }; tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { system = "x86_64-linux"; inherit nixpkgs; - inherit (self) overlay; + overlay = self.overlays.default; }; tests.nssPreload = (import ./tests/nss-preload.nix rec { system = "x86_64-linux"; inherit nixpkgs; - inherit (self) overlay; + overlay = self.overlays.default; }); tests.githubFlakes = (import ./tests/github-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; - inherit (self) overlay; + overlay = self.overlays.default; }); tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; - inherit (self) overlay; + overlay = self.overlays.default; }); tests.setuid = nixpkgs.lib.genAttrs @@ -508,7 +508,7 @@ (system: import ./tests/setuid.nix rec { inherit nixpkgs system; - inherit (self) overlay; + overlay = self.overlays.default; }); # Make sure that nix-env still produces the exact same result @@ -553,8 +553,9 @@ dockerImage = self.hydraJobs.dockerImage.${system}; }); - packages = forAllSystems (system: { + packages = forAllSystems (system: rec { inherit (nixpkgsFor.${system}) nix; + default = nix; } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; @@ -615,7 +616,7 @@ value = let nixpkgsCross = import nixpkgs { inherit system crossSystem; - overlays = [ self.overlay ]; + overlays = [ self.overlays.default ]; }; in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation { name = "nix-${version}"; @@ -655,38 +656,37 @@ nixpkgsFor.${system}."${stdenvName}Packages".nix ) stdenvs))); - defaultPackage = forAllSystems (system: self.packages.${system}.nix); + devShells = forAllSystems (system: + forAllStdenvs (stdenv: + with nixpkgsFor.${system}; + with commonDeps pkgs; + nixpkgsFor.${system}.${stdenv}.mkDerivation { + name = "nix"; - devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages); + outputs = [ "out" "dev" "doc" ]; - devShells = forAllSystemsAndStdenvs (system: stdenv: - with nixpkgsFor.${system}; - with commonDeps pkgs; + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; - nixpkgsFor.${system}.${stdenv}.mkDerivation { - name = "nix"; + inherit configureFlags; - outputs = [ "out" "dev" "doc" ]; + enableParallelBuilding = true; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; + installFlags = "sysconfdir=$(out)/etc"; - inherit configureFlags; + shellHook = + '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - }); + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + } + ) + // { default = self.devShells.${system}.stdenv; } + ); }; } From b42358b9bec12dfdc419136f32ded2a4f7d7dea7 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 18:45:58 +0200 Subject: [PATCH 17/37] Add `--exclude` flag to `nix search` If a package's attribute path, description or name contains matches for any of the regexes specified via `-e` or `--exclude` that package is excluded from the final output. --- src/nix/search.cc | 23 ++++++++++++++++++++++- src/nix/search.md | 13 ++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 87dc1c0de..62ad98999 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -18,16 +18,24 @@ using namespace nix; std::string wrap(std::string prefix, std::string s) { - return prefix + s + ANSI_NORMAL; + return concatStrings(prefix, s, ANSI_NORMAL); } struct CmdSearch : InstallableCommand, MixJSON { std::vector res; + std::vector excludeRes; CmdSearch() { expectArgs("regex", &res); + addFlag(Flag { + .longName = "exclude", + .shortName = 'e', + .description = "Hide packages whose attribute path, name or description contain *regex*.", + .labels = {"regex"}, + .handler = Handler(&excludeRes), + }); } std::string description() override @@ -62,11 +70,16 @@ struct CmdSearch : InstallableCommand, MixJSON res.push_back("^"); std::vector regexes; + std::vector excludeRegexes; regexes.reserve(res.size()); + excludeRegexes.reserve(excludeRes.size()); for (auto & re : res) regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + for (auto & re : excludeRes) + excludeRegexes.emplace_back(re, std::regex::extended | std::regex::icase); + auto state = getEvalState(); auto jsonOut = json ? std::make_unique(std::cout) : nullptr; @@ -106,6 +119,14 @@ struct CmdSearch : InstallableCommand, MixJSON std::vector nameMatches; bool found = false; + for (auto & regex : excludeRegexes) { + if ( + std::regex_search(attrPath2, regex) + || std::regex_search(name.name, regex) + || std::regex_search(description, regex)) + return; + } + for (auto & regex : regexes) { found = false; auto addAll = [&found](std::sregex_iterator it, std::vector & vec) { diff --git a/src/nix/search.md b/src/nix/search.md index d182788a6..5a5b5ae05 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -43,12 +43,23 @@ R""( # nix search nixpkgs 'firefox|chromium' ``` -* Search for packages containing `git'`and either `frontend` or `gui`: +* Search for packages containing `git` and either `frontend` or `gui`: ```console # nix search nixpkgs git 'frontend|gui' ``` +* Search for packages containing `neovim` but hide ones containing either `gui` or `python`: + + ```console + # nix search nixpkgs neovim -e 'python|gui' + ``` + or + + ```console + # nix search nixpkgs neovim -e 'python' -e 'gui' + ``` + # Description `nix search` searches *installable* (which must be evaluatable, e.g. a From e009367c8d4523bfe3a1bc20583b27d06948a390 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 18:48:48 +0200 Subject: [PATCH 18/37] Remove redundant `std::move`s in calls to `hiliteMatches` --- src/nix/search.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 62ad98999..f1f5f9641 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -154,15 +154,15 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("version", name.version); jsonElem.attr("description", description); } else { - auto name2 = hiliteMatches(name.name, std::move(nameMatches), ANSI_GREEN, "\e[0;2m"); + auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", - wrap("\e[0;1m", hiliteMatches(attrPath2, std::move(attrPathMatches), ANSI_GREEN, "\e[0;1m")), + wrap("\e[0;1m", hiliteMatches(attrPath2, attrPathMatches, ANSI_GREEN, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") logger->cout( - " %s", hiliteMatches(description, std::move(descriptionMatches), ANSI_GREEN, ANSI_NORMAL)); + " %s", hiliteMatches(description, descriptionMatches, ANSI_GREEN, ANSI_NORMAL)); } } } From 0338cf55395feb3aedabc535858263d95d235f72 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 19:44:42 +0200 Subject: [PATCH 19/37] Add tests for `--exclude` flag in `nix search` --- tests/search.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/search.sh b/tests/search.sh index 52e12f381..f320e0ec1 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -36,3 +36,9 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b (( $(nix search -f search.nix '' 'o' | grep -Eo "$e\[32;1mo{1,2}$e\[(0|0;1)m" | wc -l) == 3 )) # Searching for 'b' should yield the 'b' in bar and the two 'b's in 'broken bar' (( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) + +## Tests for --exclude +(( $(nix search -f search.nix -e hello | grep -c hello) == 0 )) + +(( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) +(( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) From 9ae22b1fdeaf6cc7541a66d981ecf7b6038739cc Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 19:45:21 +0200 Subject: [PATCH 20/37] Use `grep -c` instead of `grep|wc -l` in some `nix search` tests --- tests/search.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/search.sh b/tests/search.sh index f320e0ec1..41b706ac6 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -28,13 +28,14 @@ nix search -f search.nix '' |grep -q hello e=$'\x1b' # grep doesn't support \e, \033 or even \x1b # Multiple overlapping regexes -(( $(nix search -f search.nix '' 'oo' 'foo' 'oo' | grep "$e\[32;1mfoo$e\\[0;1m" | wc -l) == 1 )) -(( $(nix search -f search.nix '' 'broken b' 'en bar' | grep "$e\[32;1mbroken bar$e\\[0m" | wc -l) == 1 )) +(( $(nix search -f search.nix '' 'oo' 'foo' 'oo' | grep -c "$e\[32;1mfoo$e\\[0;1m") == 1 )) +(( $(nix search -f search.nix '' 'broken b' 'en bar' | grep -c "$e\[32;1mbroken bar$e\\[0m") == 1 )) # Multiple matches # Searching for 'o' should yield the 'o' in 'broken bar', the 'oo' in foo and 'o' in hello -(( $(nix search -f search.nix '' 'o' | grep -Eo "$e\[32;1mo{1,2}$e\[(0|0;1)m" | wc -l) == 3 )) +(( $(nix search -f search.nix '' 'o' | grep -Eoc "$e\[32;1mo{1,2}$e\[(0|0;1)m") == 3 )) # Searching for 'b' should yield the 'b' in bar and the two 'b's in 'broken bar' +# NOTE: This does not work with `grep -c` because it counts the two 'b's in 'broken bar' as one matched line (( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) ## Tests for --exclude From a7d25d339d94993fc8731de658f18a06e0e2a07e Mon Sep 17 00:00:00 2001 From: Jonpez2 Date: Wed, 8 Jun 2022 09:32:14 +0100 Subject: [PATCH 21/37] Add security.csm to the default ignore list --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index feb6899cd..0ee27ecb6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -802,7 +802,7 @@ public: )"}; Setting ignoredAcls{ - this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( A list of ACLs that should be ignored, normally Nix attempts to remove all ACLs from files and directories in the Nix store, but From 814ddfa5f53002216f260b3d33ca41514fa8d777 Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 8 Jun 2022 11:46:50 +0200 Subject: [PATCH 22/37] Fix missing ` in key manual --- src/nix/key-generate-secret.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md index 4938f637c..609b1abcc 100644 --- a/src/nix/key-generate-secret.md +++ b/src/nix/key-generate-secret.md @@ -30,7 +30,7 @@ convert-secret-to-public` to get the corresponding public key for verifying signed store paths. The mandatory argument `--key-name` specifies a key name (such as -`cache.example.org-1). It is used to look up keys on the client when +`cache.example.org-1`). It is used to look up keys on the client when it verifies signatures. It can be anything, but it’s suggested to use the host name of your cache (e.g. `cache.example.org`) with a suffix denoting the number of the key (to be incremented every time you need From 4a3f217bdef1b82b4f90e581e56226d18729f601 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jun 2022 13:39:44 +0200 Subject: [PATCH 23/37] Remove ${boost}/lib from the RPATH --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 217c83727..cf1ff6ee9 100644 --- a/flake.nix +++ b/flake.nix @@ -313,6 +313,7 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} From 7b968af93005348477ee19c1eb2c35937b39f249 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jun 2022 17:41:31 +0200 Subject: [PATCH 24/37] Update docker.nix Co-authored-by: Cole Helbling --- docker.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.nix b/docker.nix index a236d61d3..cbda39073 100644 --- a/docker.nix +++ b/docker.nix @@ -125,7 +125,7 @@ let sandbox = "false"; build-users-group = "nixbld"; trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; - ignored-acls = security.csm; + ignored-acls = "security.csm"; }; nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; From 931930feb139e6db0d7c01097003f8e45862f68f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 8 Jun 2022 13:45:39 -0400 Subject: [PATCH 25/37] fix(libstore/lock): support users that belong to more than 10 groups The manpage for `getgrouplist` says: > If the number of groups of which user is a member is less than or > equal to *ngroups, then the value *ngroups is returned. > > If the user is a member of more than *ngroups groups, then > getgrouplist() returns -1. In this case, the value returned in > *ngroups can be used to resize the buffer passed to a further > call getgrouplist(). In our original code, however, we allocated a list of size `10` and, if `getgrouplist` returned `-1` threw an exception. In practice, this caused the code to fail for any user belonging to more than 10 groups. While unusual for single-user systems, large companies commonly have a huge number of POSIX groups users belong to, causing this issue to crop up and make multi-user Nix unusable in such settings. The fix is relatively simple, when `getgrouplist` fails, it stores the real number of GIDs in `ngroups`, so we must resize our list and retry. Only then, if it errors once more, we can raise an exception. This should be backported to, at least, 2.9.x. --- src/libstore/lock.cc | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index f1356fdca..fa718f55d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -67,13 +67,26 @@ bool UserLock::findFreeUser() { #if __linux__ /* Get the list of supplementary groups of this build user. This is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); + int ngroups = 32; // arbitrary initial guess + supplementaryGIDs.resize(ngroups); + int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), + &ngroups); + + // Our initial size of 32 wasn't sufficient, the correct size has + // been stored in ngroups, so we try again. + if (err == -1) { + supplementaryGIDs.resize(ngroups); + err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), + &ngroups); + } + + // If it failed once more, then something must be broken. + if (err == -1) + throw Error("failed to get list of supplementary groups for '%1%'", + pw->pw_name); + + // Finally, trim back the GID list to its real size supplementaryGIDs.resize(ngroups); #endif From 3efea3d28ad522f947bacd30b74bc388c0dffa5e Mon Sep 17 00:00:00 2001 From: Sidharth Kshatriya Date: Thu, 9 Jun 2022 16:25:26 +0530 Subject: [PATCH 26/37] nix-store: small std::move() optimization --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9163eefd0..b453ea1ca 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1093,7 +1093,7 @@ static int main_nix_store(int argc, char * * argv) if (op != opDump && op != opRestore) /* !!! hack */ store = openStore(); - op(opFlags, opArgs); + op(std::move(opFlags), std::move(opArgs)); return 0; } From 7868405d58f39877a267a3f243775dd0fe92e22d Mon Sep 17 00:00:00 2001 From: Sidharth Kshatriya Date: Thu, 9 Jun 2022 19:56:36 +0530 Subject: [PATCH 27/37] nix-env: A small std::move() optimization Avoids doing a O(n) copy of Strings i.e. std::list --- src/nix-env/nix-env.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c412bb814..a69d3700d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1485,7 +1485,7 @@ static int main_nix_env(int argc, char * * argv) if (globals.profile == "") globals.profile = getDefaultProfile(); - op(globals, opFlags, opArgs); + op(globals, std::move(opFlags), std::move(opArgs)); globals.state->printStats(); From bd3a17d00cb92e114a1dc54fa3e0bac5f3261a39 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 9 Jun 2022 23:15:26 +0300 Subject: [PATCH 28/37] install-multi-user: check if selinux is enabled and if it is then abort --- scripts/install-multi-user.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index b79a9c23a..9a18280ef 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -638,6 +638,17 @@ place_channel_configuration() { fi } +check_selinux() { + if command -v getenforce > /dev/null 2>&1; then + if ! [ "$(getenforce)" = "Disabled" ]; then + failure < Date: Fri, 10 Jun 2022 09:17:28 +0100 Subject: [PATCH 29/37] Update docker.nix Co-authored-by: Eelco Dolstra --- docker.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/docker.nix b/docker.nix index cbda39073..0cd64856f 100644 --- a/docker.nix +++ b/docker.nix @@ -125,7 +125,6 @@ let sandbox = "false"; build-users-group = "nixbld"; trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; - ignored-acls = "security.csm"; }; nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; From 460117a2380c94b4e1ee514eb61e303ee283cf2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 10 Jun 2022 12:09:09 +0200 Subject: [PATCH 30/37] Correctly get the nix version in the docker job `defaultPackage` doesn't exist anymore, so we can't use it. Instead just use the new CLI which should be more robust to these changes Fix #6640 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aae5b93e0..fc6531ea5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: fetch-depth: 0 - uses: cachix/install-nix-action@v17 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV + - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 if: needs.check_cachix.outputs.secret == 'true' with: From da8f8668ca0efaad5a4134c55bf801448cec3cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 10 Jun 2022 12:57:13 +0200 Subject: [PATCH 31/37] libfetchers/git: add missing `--git-dir` flags --- src/libfetchers/git.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9cbd39247..35fdf807a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -85,8 +85,9 @@ std::optional readHead(const Path & path) bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) { Path cacheDir = getCachePath(actualUrl); + auto gitDir = "."; try { - runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef }); + runProgram("git", true, { "-C", cacheDir, "--git-dir", gitDir, "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { if (!WIFEXITED(e.status)) throw; return false; @@ -182,7 +183,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) if (hasHead) { // Using git diff is preferrable over lower-level operations here, // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"}); + auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); if (!submodules) { // Changes in submodules should only make the tree dirty // when those submodules will be copied as well. @@ -203,6 +204,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + auto gitDir = ".git"; if (!fetchSettings.allowDirty) throw Error("Git tree '%s' is dirty", workdir); @@ -210,7 +212,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co if (fetchSettings.warnDirty) warn("Git tree '%s' is dirty", workdir); - auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" }); + auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); if (submodules) gitOpts.emplace_back("--recurse-submodules"); @@ -240,7 +242,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co // modified dirty file? input.attrs.insert_or_assign( "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } From 65d09fce2216b3270499ccd8de122e197552cce6 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 10 Jun 2022 19:00:19 +0400 Subject: [PATCH 32/37] Mention that -f implies --impure for eval in docs Right now this is not mentioned anywhere and it is unexpected. --- src/libcmd/installables.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 21db2b08b..3cf25e2bc 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -146,7 +146,8 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) .shortName = 'f', .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " - "If *file* is the character -, then a Nix expression will be read from standard input.", + "If *file* is the character -, then a Nix expression will be read from standard input. " + "Implies `--impure`.", .category = installablesCategory, .labels = {"file"}, .handler = {&file}, From 754cd53faf12a9e900c7ef6cefa4a798fccea573 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 10 Jun 2022 10:49:38 -0700 Subject: [PATCH 33/37] Add missing rethrows in conditional exception handlers Signed-off-by: Anders Kaseorg --- src/libstore/gc.cc | 2 ++ src/libstore/local-binary-cache-store.cc | 1 + src/nix-collect-garbage/nix-collect-garbage.cc | 1 + 3 files changed, 4 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f65fb1b2e..d58ed78b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -135,6 +135,7 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdRootsSocket.close(); goto restart; } + throw; } } @@ -153,6 +154,7 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdRootsSocket.close(); goto restart; } + throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); state->fdRootsSocket.close(); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a3c3e4806..ba4416f6d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -69,6 +69,7 @@ protected: } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + throw; } } diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index af6f1c88c..e413faffe 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -37,6 +37,7 @@ void removeOldGenerations(std::string dir) link = readLink(path); } catch (SysError & e) { if (e.errNo == ENOENT) continue; + throw; } if (link.find("link") != std::string::npos) { printInfo(format("removing old generations of profile %1%") % path); From 502d7d9092ccf792a27088f31571dbace96f1962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 11 Jun 2022 15:13:58 +0200 Subject: [PATCH 34/37] nix-build: stop logger when appropriate Reverts b944b588fa280b0555b8269c0f6d097352f8716f in `nix-build.cc`. --- src/nix-build/nix-build.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 426f23905..519855ea3 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -543,6 +543,8 @@ static void main_nix_build(int argc, char * * argv) restoreProcessContext(); + logger->stop(); + execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -601,6 +603,8 @@ static void main_nix_build(int argc, char * * argv) outPaths.push_back(outputPath); } + logger->stop(); + for (auto & path : outPaths) std::cout << store->printStorePath(path) << '\n'; } From 9f6b4639c2060aa6d7f7336222dad4ea350ccdf8 Mon Sep 17 00:00:00 2001 From: Gabriel Fontes Date: Sat, 11 Jun 2022 16:52:20 -0300 Subject: [PATCH 35/37] fix sourcehut brach/tag resolving regression nixos/nix#6290 introduced a regex pattern to account for tags when resolving sourcehut refs. nixos/nix#4638 reafactored the code, accidentally treating the pattern as a regular string, causing all non-HEAD ref resolving to break. This fixes the regression and adds more test cases to avoid future breakage. --- src/libfetchers/github.cc | 9 +++++---- tests/sourcehut-flakes.nix | 13 ++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0631fb6e8..a491d82a6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -381,7 +381,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - std::string ref_uri; + std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); @@ -393,10 +393,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme if (!remoteLine) { throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); } - ref_uri = remoteLine->target; + refUri = remoteLine->target; } else { - ref_uri = fmt("refs/(heads|tags)/%s", ref); + refUri = fmt("refs/(heads|tags)/%s", ref); } + std::regex refRegex(refUri); auto file = store->toRealPath( downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); @@ -406,7 +407,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::optional id; while(!id && getline(is, line)) { auto parsedLine = git::parseLsRemoteLine(line); - if (parsedLine && parsedLine->reference == ref_uri) + if (parsedLine && parsedLine->reference && std::regex_match(*parsedLine->reference, refRegex)) id = parsedLine->target; } diff --git a/tests/sourcehut-flakes.nix b/tests/sourcehut-flakes.nix index aadab9bb5..daa259dd6 100644 --- a/tests/sourcehut-flakes.nix +++ b/tests/sourcehut-flakes.nix @@ -59,7 +59,7 @@ let echo 'ref: refs/heads/master' > $out/HEAD mkdir -p $out/info - echo -e '${nixpkgs.rev}\trefs/heads/master' > $out/info/refs + echo -e '${nixpkgs.rev}\trefs/heads/master\n${nixpkgs.rev}\trefs/tags/foo-bar' > $out/info/refs ''; in @@ -132,6 +132,17 @@ makeTest ( client.succeed("curl -v https://git.sr.ht/ >&2") client.succeed("nix registry list | grep nixpkgs") + # Test that it resolves HEAD + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + # Test that it resolves branches + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs/master --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + # Test that it resolves tags + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs/foo-bar --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + + # Registry and pinning test rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" From d82a3dc70d5a5c68815327a8922c8db0d0c95cdb Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Mon, 13 Jun 2022 20:49:16 +0400 Subject: [PATCH 36/37] flake.cc: Make non-flake overrides sticky Overrides for inputs with flake=false were non-sticky, since they changed the `original` in `flake.lock`. This fixes it, by using the same locked original for both flake and non-flake inputs. --- src/libexpr/flake/flake.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 35c841897..920726b73 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -513,6 +513,15 @@ LockedFlake lockFlake( if (!lockFlags.allowMutable && !input.ref->input.isLocked()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto ref = input2.ref ? *input2.ref : *input.ref; + if (input.isFlake) { Path localPath = parentPath; FlakeRef localRef = *input.ref; @@ -524,15 +533,7 @@ LockedFlake lockFlake( auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto childNode = std::make_shared( - inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + auto childNode = std::make_shared(inputFlake.lockedRef, ref); node->inputs.insert_or_assign(id, childNode); @@ -560,7 +561,7 @@ LockedFlake lockFlake( auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, *input.ref, false)); + std::make_shared(lockedRef, ref, false)); } } From fd7f795750ad97466c0fb9733ac3cc2a0909b84d Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Fri, 3 Jun 2022 23:19:12 +0200 Subject: [PATCH 37/37] Add disambiguation to man page This should help future lost newcomers like myself understand where to find the docs for both of these commands and how they differ. --- doc/manual/src/command-ref/nix-build.md | 6 ++++++ doc/manual/src/command-ref/nix-shell.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index aacb32a25..49c6f3f55 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -12,6 +12,12 @@ [`--dry-run`] [{`--out-link` | `-o`} *outlink*] +# Disambiguation + +This man page describes the command `nix-build`, which is distinct from `nix +build`. For documentation on the latter, run `nix build --help` or see `man +nix3-build`. + # Description The `nix-build` command builds the derivations described by the Nix diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index a2b6d8a8e..840bccd25 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -15,6 +15,12 @@ [`--keep` *name*] {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]} +# Disambiguation + +This man page describes the command `nix-shell`, which is distinct from `nix +shell`. For documentation on the latter, run `nix shell --help` or see `man +nix3-shell`. + # Description The command `nix-shell` will build the dependencies of the specified