From 15833516a4bad0a4ae7786293b22df4bf650aa80 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 5 Feb 2019 16:42:45 -0500 Subject: [PATCH 001/555] Add armv6l-linux & armv7l-linux as cross jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a cheap way to get 32-bit ARM working. We don’t support it officially but lots of people have raspberry pis and similar hardware they want to install the Nix package manager on. --- release.nix | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/release.nix b/release.nix index 271645067..0ec742906 100644 --- a/release.nix +++ b/release.nix @@ -2,6 +2,7 @@ , nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, crossSystems ? [ "armv6l-linux" "armv7l-linux" ] }: let @@ -53,11 +54,12 @@ let }; - build = pkgs.lib.genAttrs systems (system: + build = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; with import ./release-common.nix { inherit pkgs; }; @@ -89,9 +91,12 @@ let }); - perlBindings = pkgs.lib.genAttrs systems (system: + perlBindings = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; releaseTools.nixBuild { name = "nix-perl"; @@ -112,9 +117,12 @@ let }); - binaryTarball = pkgs.lib.genAttrs systems (system: + binaryTarball = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - with import nixpkgs { inherit system; }; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; let toplevel = builtins.getAttr system jobs.build; From 1996af425ac8ddea1e8a591650e7d0caba2aa201 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 21:43:47 -0500 Subject: [PATCH 002/555] Use buildPackages for native dependencies Unfortunately, releaseTools.nixBuild does not separate native and non-native build inputs. As an alternative, we can just use buildPackages to get the native version of some packages like: - pkgconfig - git - curl - utillinux --- release-common.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release-common.nix b/release-common.nix index 4c5565985..f2aa57c2c 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,14 +50,16 @@ rec { buildDeps = [ curl bzip2 xz brotli editline - openssl pkgconfig sqlite boehmgc + openssl sqlite boehmgc boost + buildPackages.pkgconfig + # Tests - git - mercurial + buildPackages.git + buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { From e9072ded9749ab00cc397980e8a26f83d341efc0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 22:43:28 -0500 Subject: [PATCH 003/555] Use nativeBuildInputs --- release-common.nix | 16 +++++++++------- release.nix | 3 +++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/release-common.nix b/release-common.nix index f2aa57c2c..707d36f95 100644 --- a/release-common.nix +++ b/release-common.nix @@ -47,19 +47,21 @@ rec { autoreconfHook ]; + nativeBuildDeps = + [ buildPackages.pkgconfig + + # Tests + buildPackages.git + buildPackages.mercurial + ] ++ lib.optional stdenv.isLinux buildPackages.utillinuxMinimal; + buildDeps = [ curl bzip2 xz brotli editline openssl sqlite boehmgc boost - - buildPackages.pkgconfig - - # Tests - buildPackages.git - buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { diff --git a/release.nix b/release.nix index 0ec742906..9843efa29 100644 --- a/release.nix +++ b/release.nix @@ -24,6 +24,7 @@ let src = nix; inherit officialRelease; + nativeBuildInputs = nativeBuildDeps; buildInputs = tarballDeps ++ buildDeps; configureFlags = "--enable-gc"; @@ -67,6 +68,7 @@ let name = "nix"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; preConfigure = @@ -199,6 +201,7 @@ let name = "nix-build"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; dontInstall = false; From f6ea56dfac1f4df45a5fa9e2801bc632dee9eff7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 23:04:40 -0500 Subject: [PATCH 004/555] Get shellcheck from buildPackages --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 9843efa29..06db7bdc0 100644 --- a/release.nix +++ b/release.nix @@ -133,7 +133,7 @@ let in runCommand "nix-binary-tarball-${version}" - { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + { nativeBuildInputs = lib.optional (system != "aarch64-linux") buildPackages.shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' From 4fefe26717fa70828e3f524e43c76e3f7b7a09b0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 5 Feb 2021 18:22:34 -0600 Subject: [PATCH 005/555] Re-enable armv6l support This fixes the libatomic detection. --- configure.ac | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 2047ed8d2..685c471c5 100644 --- a/configure.ac +++ b/configure.ac @@ -152,7 +152,7 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="$LDFLAGS -latomic" fi PKG_PROG_PKG_CONFIG diff --git a/flake.nix b/flake.nix index 869b92cb7..7e02fd70d 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" ]; - crossSystems = [ "armv7l-linux" ]; + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); From 45473d02c95cdf20c782d5814f22fc1de5e36b2f Mon Sep 17 00:00:00 2001 From: p01arst0rm Date: Fri, 23 Apr 2021 07:30:05 +0100 Subject: [PATCH 006/555] unified macro style for ENABLE_S3 --- configure.ac | 4 ++-- src/libstore/filetransfer.cc | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 6c36787f3..05523eef8 100644 --- a/configure.ac +++ b/configure.ac @@ -235,8 +235,8 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) AC_CHECK_HEADERS([aws/s3/S3Client.h], - [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) - enable_s3=1], [enable_s3=]) + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], + [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=]) AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 514ab3bf9..2cf35ec83 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -7,7 +7,7 @@ #include "finally.hh" #include "callback.hh" -#ifdef ENABLE_S3 +#if ENABLE_S3 #include #endif @@ -665,7 +665,7 @@ struct curlFileTransfer : public FileTransfer writeFull(wakeupPipe.writeSide.get(), " "); } -#ifdef ENABLE_S3 +#if ENABLE_S3 std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); @@ -688,7 +688,7 @@ struct curlFileTransfer : public FileTransfer if (hasPrefix(request.uri, "s3://")) { // FIXME: do this on a worker thread try { -#ifdef ENABLE_S3 +#if ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); std::string profile = get(params, "profile").value_or(""); From 7dca9c24c4acdf2e2abe0f6a138b734701098cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Tue, 27 Apr 2021 10:44:29 +0200 Subject: [PATCH 007/555] nix build: make dry-run to print a json output if --json is enabled --- src/nix/build.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 03159b6cc..226c551fa 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -54,6 +54,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); + if (dryRun) return; if (outLink != "") @@ -79,8 +81,6 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile } updateProfile(buildables); - - if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); } }; From eab14a642cbcbc35f4473888d906f9de7deda07b Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 27 Nov 2020 16:42:15 -0600 Subject: [PATCH 008/555] darwin: encrypt nix volume if filevault is enabled --- .../src/installation/installing-binary.md | 203 +--- scripts/create-darwin-volume.sh | 900 +++++++++++++++--- scripts/install-darwin-multi-user.sh | 112 ++- scripts/install-multi-user.sh | 289 ++++-- scripts/install-nix-from-closure.sh | 91 +- scripts/install-systemd-multi-user.sh | 25 +- 6 files changed, 1186 insertions(+), 434 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index ae7fd458b..96fa34635 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,18 +1,26 @@ # Installing a Binary Distribution -If you are using Linux or macOS versions up to 10.14 (Mojave), the -easiest way to install Nix is to run the following command: +The easiest way to install Nix is to run the following command: ```console $ sh <(curl -L https://nixos.org/nix/install) ``` -If you're using macOS 10.15 (Catalina) or newer, consult [the macOS -installation instructions](#macos-installation) before installing. +This will run the installer interactively (causing it to explain what +it is doing more explicitly), and perform the default "type" of install +for your platform: +- single-user on Linux +- multi-user on macOS -As of Nix 2.1.0, the Nix installer will always default to creating a -single-user installation, however opting in to the multi-user -installation is highly recommended. + > **Notes on read-only filesystem root in macOS 10.15 Catalina +** + > + > - It took some time to support this cleanly. You may see posts, + > examples, and tutorials using obsolete workarounds. + > - Supporting it cleanly made macOS installs too complex to qualify + > as single-user, so this type is no longer supported on macOS. + +We recommend the multi-user install if it supports your platform and +you can authenticate with `sudo`. # Single User Installation @@ -50,9 +58,9 @@ $ rm -rf /nix The multi-user Nix installation creates system users, and a system service for the Nix daemon. - - Linux running systemd, with SELinux disabled - - - macOS +**Supported Systems** +- Linux running systemd, with SELinux disabled +- macOS You can instruct the installer to perform a multi-user installation on your system: @@ -96,165 +104,28 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist There may also be references to Nix in `/etc/profile`, `/etc/bashrc`, and `/etc/zshrc` which you may remove. -# macOS Installation +# macOS Installation + -Starting with macOS 10.15 (Catalina), the root filesystem is read-only. -This means `/nix` can no longer live on your system volume, and that -you'll need a workaround to install Nix. +We believe we have ironed out how to cleanly support the read-only root +on modern macOS. New installs will do this automatically, and you can +also re-run a new installer to convert your existing setup. -The recommended approach, which creates an unencrypted APFS volume for -your Nix store and a "synthetic" empty directory to mount it over at -`/nix`, is least likely to impair Nix or your system. +This section previously detailed the situation, options, and trade-offs, +but it now only outlines what the installer does. You don't need to know +this to run the installer, but it may help if you run into trouble: -> **Note** -> -> With all separate-volume approaches, it's possible something on your -> system (particularly daemons/services and restored apps) may need -> access to your Nix store before the volume is mounted. Adding -> additional encryption makes this more likely. - -If you're using a recent Mac with a [T2 -chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf), -your drive will still be encrypted at rest (in which case "unencrypted" -is a bit of a misnomer). To use this approach, just install Nix with: - -```console -$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume -``` - -If you don't like the sound of this, you'll want to weigh the other -approaches and tradeoffs detailed in this section. - -> **Note** -> -> All of the known workarounds have drawbacks, but we hope better -> solutions will be available in the future. Some that we have our eye -> on are: -> -> 1. A true firmlink would enable the Nix store to live on the primary -> data volume without the build problems caused by the symlink -> approach. End users cannot currently create true firmlinks. -> -> 2. If the Nix store volume shared FileVault encryption with the -> primary data volume (probably by using the same volume group and -> role), FileVault encryption could be easily supported by the -> installer without requiring manual setup by each user. - -## Change the Nix store path prefix - -Changing the default prefix for the Nix store is a simple approach which -enables you to leave it on your root volume, where it can take full -advantage of FileVault encryption if enabled. Unfortunately, this -approach also opts your device out of some benefits that are enabled by -using the same prefix across systems: - - - Your system won't be able to take advantage of the binary cache - (unless someone is able to stand up and support duplicate caching - infrastructure), which means you'll spend more time waiting for - builds. - - - It's harder to build and deploy packages to Linux systems. - -It would also possible (and often requested) to just apply this change -ecosystem-wide, but it's an intrusive process that has side effects we -want to avoid for now. - -## Use a separate encrypted volume - -If you like, you can also add encryption to the recommended approach -taken by the installer. You can do this by pre-creating an encrypted -volume before you run the installer--or you can run the installer and -encrypt the volume it creates later. - -In either case, adding encryption to a second volume isn't quite as -simple as enabling FileVault for your boot volume. Before you dive in, -there are a few things to weigh: - -1. The additional volume won't be encrypted with your existing - FileVault key, so you'll need another mechanism to decrypt the - volume. - -2. You can store the password in Keychain to automatically decrypt the - volume on boot--but it'll have to wait on Keychain and may not mount - before your GUI apps restore. If any of your launchd agents or apps - depend on Nix-installed software (for example, if you use a - Nix-installed login shell), the restore may fail or break. - - On a case-by-case basis, you may be able to work around this problem - by using `wait4path` to block execution until your executable is - available. - - It's also possible to decrypt and mount the volume earlier with a - login hook--but this mechanism appears to be deprecated and its - future is unclear. - -3. You can hard-code the password in the clear, so that your store - volume can be decrypted before Keychain is available. - -If you are comfortable navigating these tradeoffs, you can encrypt the -volume with something along the lines of: - -```console -$ diskutil apfs enableFileVault /nix -user disk -``` - -## Symlink the Nix store to a custom location - -Another simple approach is using `/etc/synthetic.conf` to symlink the -Nix store to the data volume. This option also enables your store to -share any configured FileVault encryption. Unfortunately, builds that -resolve the symlink may leak the canonical path or even fail. - -Because of these downsides, we can't recommend this approach. - -## Notes on the recommended approach - -This section goes into a little more detail on the recommended approach. -You don't need to understand it to run the installer, but it can serve -as a helpful reference if you run into trouble. - -1. In order to compose user-writable locations into the new read-only - system root, Apple introduced a new concept called `firmlinks`, - which it describes as a "bi-directional wormhole" between two - filesystems. You can see the current firmlinks in - `/usr/share/firmlinks`. Unfortunately, firmlinks aren't (currently?) - user-configurable. - - For special cases like NFS mount points or package manager roots, - [synthetic.conf(5)](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html) - supports limited user-controlled file-creation (of symlinks, and - synthetic empty directories) at `/`. To create a synthetic empty - directory for mounting at `/nix`, add the following line to - `/etc/synthetic.conf` (create it if necessary): - - nix - -2. This configuration is applied at boot time, but you can use - `apfs.util` to trigger creation (not deletion) of new entries - without a reboot: - - ```console - $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B - ``` - -3. Create the new APFS volume with diskutil: - - ```console - $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix - ``` - -4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't - already have other entries, it should look something like: - - # - # Warning - this file should only be modified with vifs(8) - # - # Failure to do so is unsupported and may be destructive. - # - LABEL=Nix\040Store /nix apfs rw,nobrowse - - The nobrowse setting will keep Spotlight from indexing this volume, - and keep it from showing up on your desktop. +- create a new APFS volume for your Nix store +- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" + empty root directory to mount your volume +- specify mount options for the volume in `/etc/fstab` +- if you have FileVault enabled + - generate an encryption password + - put it in your system Keychain + - use it to encrypt the volume +- create a system LaunchDaemon to mount this volume early enough in the + boot process to avoid problems loading or restoring any programs that + need access to your Nix store # Installing a pinned Nix version from a URL diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 32fa577a8..8aff03199 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -1,33 +1,262 @@ -#!/bin/sh -set -e +#!/usr/bin/env bash +set -eu +set -o pipefail -root_disk() { - diskutil info -plist / -} +# I'm a little agnostic on the choices, but supporting a wide +# slate of uses for now, including: +# - import-only: `. create-darwin-volume.sh no-main[ ...]` +# - legacy: `./create-darwin-volume.sh` or `. create-darwin-volume.sh` +# (both will run main()) +# - external alt-routine: `./create-darwin-volume.sh no-main func[ ...]` +if [ "${1-}" = "no-main" ]; then + shift + readonly _CREATE_VOLUME_NO_MAIN=1 +else + readonly _CREATE_VOLUME_NO_MAIN=0 + # declare some things we expect to inherit from install-multi-user + # I don't love this (because it's a bit of a kludge). + # + # CAUTION: (Dec 19 2020) + # This is a stopgap. It doesn't cover the full slate of + # identifiers we inherit--just those necessary to: + # - avoid breaking direct invocations of this script (here/now) + # - avoid hard-to-reverse structural changes before the call to rm + # single-user support is verified + # + # In the near-mid term, I (personally) think we should: + # - decide to deprecate the direct call and add a notice + # - fold all of this into install-darwin-multi-user.sh + # - intentionally remove the old direct-invocation form (kill the + # routine, replace this script w/ deprecation notice and a note + # on the remove-after date) + # + readonly NIX_ROOT="${NIX_ROOT:-/nix}" -# i.e., "disk1" + _sudo() { + shift # throw away the 'explanation' + /usr/bin/sudo "$@" + } + failure() { + if [ "$*" = "" ]; then + cat + else + echo "$@" + fi + exit 1 + } + task() { + echo "$@" + } +fi + +# usually "disk1" root_disk_identifier() { - diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # For performance (~10ms vs 280ms) I'm parsing 'diskX' from stat output + # (~diskXsY)--but I'm retaining the more-semantic approach since + # it documents intent better. + # /usr/sbin/diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # + local special_device + special_device="$(/usr/bin/stat -f "%Sd" /)" + echo "${special_device%s[0-9]*}" } -find_nix_volume() { - diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true +# make it easy to play w/ 'Case-sensitive APFS' +readonly NIX_VOLUME_FS="${NIX_VOLUME_FS:-APFS}" +readonly NIX_VOLUME_LABEL="${NIX_VOLUME_LABEL:-Nix Store}" +# Strongly assuming we'll make a volume on the device / is on +# But you can override NIX_VOLUME_USE_DISK to create it on some other device +readonly NIX_VOLUME_USE_DISK="${NIX_VOLUME_USE_DISK:-$(root_disk_identifier)}" +NIX_VOLUME_USE_SPECIAL="${NIX_VOLUME_USE_SPECIAL:-}" +NIX_VOLUME_USE_UUID="${NIX_VOLUME_USE_UUID:-}" +readonly NIX_VOLUME_MOUNTD_DEST="${NIX_VOLUME_MOUNTD_DEST:-/Library/LaunchDaemons/org.nixos.darwin-store.plist}" + +if /usr/bin/fdesetup isactive >/dev/null; then + test_filevault_in_use() { return 0; } + # no readonly; we may modify if user refuses from cure_volume + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-1}" +else + test_filevault_in_use() { return 1; } + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-0}" +fi + +should_encrypt_volume() { + test_filevault_in_use && (( NIX_VOLUME_DO_ENCRYPT == 1 )) +} + +substep() { + printf " %s\n" "" "- $1" "" "${@:2}" +} + + +volumes_labeled() { + local label="$1" + xsltproc --novalid --stringparam label "$label" - <(/usr/sbin/ioreg -ra -c "AppleAPFSVolume") <<'EOF' + + + + + + + + = + + + + +EOF + # I cut label out of the extracted values, but here it is for reference: + # + # = +} + +right_disk() { + local volume_special="$1" # (i.e., disk1s7) + [[ "$volume_special" == "$NIX_VOLUME_USE_DISK"s* ]] +} + +right_volume() { + local volume_special="$1" # (i.e., disk1s7) + # if set, it must match; otherwise ensure it's on the right disk + if [ -z "$NIX_VOLUME_USE_SPECIAL" ]; then + if right_disk "$volume_special"; then + NIX_VOLUME_USE_SPECIAL="$volume_special" # latch on + return 0 + else + return 1 + fi + else + [ "$volume_special" = "$NIX_VOLUME_USE_SPECIAL" ] + fi +} + +right_uuid() { + local volume_uuid="$1" + # if set, it must match; otherwise allow + if [ -z "$NIX_VOLUME_USE_UUID" ]; then + NIX_VOLUME_USE_UUID="$volume_uuid" # latch on + return 0 + else + [ "$volume_uuid" = "$NIX_VOLUME_USE_UUID" ] + fi +} + +cure_volumes() { + local found volume special uuid + # loop just in case they have more than one volume + # (nothing stops you from doing this) + for volume in $(volumes_labeled "$NIX_VOLUME_LABEL"); do + # CAUTION: this could (maybe) be a more normal read + # loop like: + # while IFS== read -r special uuid; do + # # ... + # done <<<"$(volumes_labeled "$NIX_VOLUME_LABEL")" + # + # I did it with for to skirt a problem with the obvious + # pattern replacing stdin and causing user prompts + # inside (which also use read and access stdin) to skip + # + # If there's an existing encrypted volume we can't find + # in keychain, the user never gets prompted to delete + # the volume, and the install fails. + # + # If you change this, a human needs to test a very + # specific scenario: you already have an encrypted + # Nix Store volume, and have deleted its credential + # from keychain. Ensure the script asks you if it can + # delete the volume, and then prompts for your sudo + # password to confirm. + # + # shellcheck disable=SC1097 + IFS== read -r special uuid <<< "$volume" + # take the first one that's on the right disk + if [ -z "${found:-}" ]; then + if right_volume "$special" && right_uuid "$uuid"; then + cure_volume "$special" "$uuid" + found="${special} (${uuid})" + else + warning < + # Cryptographic user for (1 found) + # Cryptographic users for (2 found) + /usr/sbin/diskutil apfs listCryptoUsers -plist "$volume_special" | /usr/bin/grep -q APFSCryptoUserUUID } test_fstab() { - grep -q "/nix apfs rw" /etc/fstab 2>/dev/null + /usr/bin/grep -q "$NIX_ROOT apfs rw" /etc/fstab 2>/dev/null } -test_nix_symlink() { - [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null +test_nix_root_is_symlink() { + [ -L "$NIX_ROOT" ] } -test_synthetic_conf() { - grep -q "^nix$" /etc/synthetic.conf 2>/dev/null +test_synthetic_conf_either(){ + /usr/bin/grep -qE "^${NIX_ROOT:1}($|\t.{3,}$)" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_mountable() { + /usr/bin/grep -q "^${NIX_ROOT:1}$" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_symlinked() { + /usr/bin/grep -qE "^${NIX_ROOT:1}\t.{3,}$" /etc/synthetic.conf 2>/dev/null +} + +test_nix_volume_mountd_installed() { + test -e "$NIX_VOLUME_MOUNTD_DEST" +} + +# current volume password +test_keychain_by_uuid() { + local volume_uuid="$1" + # Note: doesn't need sudo just to check; doesn't output pw + security find-generic-password -s "$volume_uuid" &>/dev/null +} + +get_volume_pass() { + local volume_uuid="$1" + _sudo \ + "to confirm keychain has a password that unlocks this volume" \ + security find-generic-password -s "$volume_uuid" -w +} + +verify_volume_pass() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid" +} + +volume_pass_works() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + get_volume_pass "$volume_uuid" | verify_volume_pass "$volume_special" "$volume_uuid" } # Create the paths defined in synthetic.conf, saving us a reboot. -create_synthetic_objects(){ +create_synthetic_objects() { # Big Sur takes away the -B flag we were using and replaces it # with a -t flag that appears to do the same thing (but they # don't behave exactly the same way in terms of return values). @@ -41,129 +270,570 @@ create_synthetic_objects(){ } test_nix() { - test -d "/nix" + test -d "$NIX_ROOT" } -test_t2_chip_present(){ - # Use xartutil to see if system has a t2 chip. - # - # This isn't well-documented on its own; until it is, - # let's keep track of knowledge/assumptions. - # - # Warnings: - # - Don't search "xart" if porn will cause you trouble :) - # - Other xartutil flags do dangerous things. Don't run them - # naively. If you must, search "xartutil" first. - # - # Assumptions: - # - the "xART session seeds recovery utility" - # appears to interact with xartstorageremoted - # - `sudo xartutil --list` lists xART sessions - # and their seeds and exits 0 if successful. If - # not, it exits 1 and prints an error such as: - # xartutil: ERROR: No supported link to the SEP present - # - xART sessions/seeds are present when a T2 chip is - # (and not, otherwise) - # - the presence of a T2 chip means a newly-created - # volume on the primary drive will be - # encrypted at rest - # - all together: `sudo xartutil --list` - # should exit 0 if a new Nix Store volume will - # be encrypted at rest, and exit 1 if not. - sudo xartutil --list >/dev/null 2>/dev/null +test_voldaemon() { + test -f "$NIX_VOLUME_MOUNTD_DEST" } -test_filevault_in_use() { - fdesetup isactive >/dev/null +generate_mount_command() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid mountpoint cmd=() + printf -v volume_uuid "%q" "$2" + printf -v mountpoint "%q" "$NIX_ROOT" + + case "$cmd_type" in + encrypted) + cmd=(/bin/sh -c "/usr/bin/security find-generic-password -s '$volume_uuid' -w | /usr/sbin/diskutil apfs unlockVolume '$volume_uuid' -mountpoint '$mountpoint' -stdinpassphrase");; + unencrypted) + cmd=(/usr/sbin/diskutil mount -mountPoint "$mountpoint" "$volume_uuid");; + *) + failure "Invalid first arg $cmd_type to generate_mount_command";; + esac + + printf " %s\n" "${cmd[@]}" } -# use after error msg for conditions we don't understand -suggest_report_error(){ - # ex "error: something sad happened :(" >&2 - echo " please report this @ https://github.com/nixos/nix/issues" >&2 +generate_mount_daemon() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid="$2" + cat < + + + + RunAtLoad + + Label + org.nixos.darwin-store + ProgramArguments + +$(generate_mount_command "$cmd_type" "$volume_uuid") + + + +EOF } -main() { - ( - echo "" - echo " ------------------------------------------------------------------ " - echo " | This installer will create a volume for the nix store and |" - echo " | configure it to mount at /nix. Follow these steps to uninstall. |" - echo " ------------------------------------------------------------------ " - echo "" - echo " 1. Remove the entry from fstab using 'sudo vifs'" - echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" - echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" - echo "" - ) >&2 +_eat_bootout_err() { + /usr/bin/grep -v "Boot-out failed: 36: Operation now in progress" +} - if test_nix_symlink; then - echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 - echo " /nix -> $(readlink "/nix")" >&2 - exit 2 +# TODO: remove with --uninstall? +uninstall_launch_daemon_directions() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + substep "Uninstall LaunchDaemon $daemon_label" \ + " sudo launchctl bootout system/$daemon_label" \ + " sudo rm $daemon_plist" +} + +uninstall_launch_daemon_prompt() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + local reason_for_daemon="$3" + cat < >(_eat_bootout_err >&2) || true + # this can "fail" with a message like: + # Boot-out failed: 36: Operation now in progress + _sudo "to remove the daemon definition" rm "$daemon_plist" + fi +} + +nix_volume_mountd_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" +} + +nix_volume_mountd_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" \ + "mount your Nix volume" +} + +# TODO: move nix_daemon to install-darwin-multi-user if/when uninstall_launch_daemon_prompt moves up to install-multi-user +nix_daemon_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" \ + "run the nix-daemon" +} + +# TODO: remove with --uninstall? +nix_daemon_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" +} + + +# TODO: remove with --uninstall? +synthetic_conf_uninstall_directions() { + # :1 to strip leading slash + substep "Remove ${NIX_ROOT:1} from /etc/synthetic.conf" \ + " If nix is the only entry: sudo rm /etc/synthetic.conf" \ + " Otherwise: sudo /usr/bin/sed -i '' -e '/^${NIX_ROOT:1}$/d' /etc/synthetic.conf" +} + +synthetic_conf_uninstall_prompt() { + cat < "$SCRATCH/synthetic.conf.edit" + + if test_synthetic_conf_symlinked; then + warning <&2 - echo nix | sudo tee -a /etc/synthetic.conf - if ! test_synthetic_conf; then - echo "error: failed to configure synthetic.conf;" >&2 - suggest_report_error - exit 1 + # ask to rm if this left the file empty aside from comments, else edit + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/synthetic.conf.edit") &>/dev/null; then + if confirm_rm "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 + fi + else + if confirm_edit "$SCRATCH/synthetic.conf.edit" "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 fi fi + # fallback instructions + echo "Manually remove nix from /etc/synthetic.conf" + return 1 +} - if ! test_nix; then - echo "Creating mountpoint for /nix..." >&2 - create_synthetic_objects # the ones we defined in synthetic.conf - if ! test_nix; then - sudo mkdir -p /nix 2>/dev/null || true +add_nix_vol_fstab_line() { + local uuid="$1" + # shellcheck disable=SC1003,SC2026 + local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" + shift + EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <multi-user reinstalls, which may cover this) + # + # I'm not sure if it's safe to approach this way? + # + # I think I think the most-proper way to test for it is: + # diskutil info -plist "$NIX_VOLUME_LABEL" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1][name()='true']" -; echo $? + # + # There's also `sudo /usr/sbin/vsdbutil -c /path` (which is much faster, but is also + # deprecated and needs minor parsing). + # + # If no one finds a problem with doing so, I think the simplest approach + # is to just eagerly set this. I found a few imperative approaches: + # (diskutil enableOwnership, ~100ms), a cheap one (/usr/sbin/vsdbutil -a, ~40-50ms), + # a very cheap one (append the internal format to /var/db/volinfo.database). + # + # But vsdbutil's deprecation notice suggests using fstab, so I want to + # give that a whirl first. + # + # TODO: when this is workable, poke infinisil about reproducing the issue + # and confirming this fix? +} + +delete_nix_vol_fstab_line() { + # TODO: I'm scaffolding this to handle the new nix volumes + # but it might be nice to generalize a smidge further to + # go ahead and set up a pattern for curing "old" things + # we no longer do? + EDITOR="/usr/bin/patch" _sudo "to cut nix from fstab" "$@" < <(/usr/bin/diff /etc/fstab <(/usr/bin/grep -v "$NIX_ROOT apfs rw" /etc/fstab)) + # leaving some parts out of the grep; people may fiddle this a little? +} + +# TODO: hope to remove with --uninstall +fstab_uninstall_directions() { + substep "Remove ${NIX_ROOT} from /etc/fstab" \ + " If nix is the only entry: sudo rm /etc/fstab" \ + " Otherwise, run 'sudo /usr/sbin/vifs' to remove the nix line" +} + +fstab_uninstall_prompt() { + cat </dev/null + + # if the patch test edit, minus comment lines, is equal to empty (:) + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/fstab.edit") &>/dev/null; then + # this edit would leave it empty; propose deleting it + if confirm_rm "/etc/fstab"; then + return 0 + else + echo "Remove nix from /etc/fstab (or remove the file)" fi - if ! test_nix; then - echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 - suggest_report_error - exit 1 + else + echo "I might be able to help you make this edit. Here's the diff:" + if ! _diff "/etc/fstab" "$SCRATCH/fstab.edit" && ui_confirm "Does the change above look right?"; then + delete_nix_vol_fstab_line /usr/sbin/vifs + else + echo "Remove nix from /etc/fstab (or remove the file)" fi fi +} - disk="$(root_disk_identifier)" - volume=$(find_nix_volume "$disk") - if [ -z "$volume" ]; then - echo "Creating a Nix Store volume..." >&2 +remove_volume() { + local volume_special="$1" # (i.e., disk1s7) + _sudo "to unmount the Nix volume" \ + /usr/sbin/diskutil unmount force "$volume_special" || true # might not be mounted + _sudo "to delete the Nix volume" \ + /usr/sbin/diskutil apfs deleteVolume "$volume_special" +} - if test_filevault_in_use; then - # TODO: Not sure if it's in-scope now, but `diskutil apfs list` - # shows both filevault and encrypted at rest status, and it - # may be the more semantic way to test for this? It'll show - # `FileVault: No (Encrypted at rest)` - # `FileVault: No` - # `FileVault: Yes (Unlocked)` - # and so on. - if test_t2_chip_present; then - echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 - echo " is only encrypted at rest." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 +# aspiration: robust enough to both fix problems +# *and* update older darwin volumes +cure_volume() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + header "Found existing Nix volume" + row " special" "$volume_special" + row " uuid" "$volume_uuid" + + if volume_encrypted "$volume_special"; then + row "encrypted" "yes" + if volume_pass_works "$volume_special" "$volume_uuid"; then + NIX_VOLUME_DO_ENCRYPT=0 + ok "Found a working decryption password in keychain :)" + echo "" + else + # - this is a volume we made, and + # - the user encrypted it on their own + # - something deleted the credential + # - this is an old or BYO volume and the pw + # just isn't somewhere we can find it. + # + # We're going to explain why we're freaking out + # and prompt them to either delete the volume + # (requiring a sudo auth), or abort to fix + warning <&2 - echo " FileVault encrypted, but encryption-at-rest is not available." >&2 - echo " Manually create a volume for the store and re-run this script." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 - exit 1 + # TODO: this is a good design case for a warn-and + # remind idiom... + failure <&2 - fi - - if ! test_fstab; then - echo "Configuring /etc/fstab..." >&2 - label=$(echo "$volume" | sed 's/ /\\040/g') - # shellcheck disable=SC2209 - printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + row "encrypted" "no" fi } -main "$@" +remove_volume_artifacts() { + if test_synthetic_conf_either; then + # NIX_ROOT is in synthetic.conf + if synthetic_conf_uninstall_prompt; then + # TODO: moot until we tackle uninstall, but when we're + # actually uninstalling, we should issue: + # reminder "macOS will clean up the empty mount-point directory at $NIX_ROOT on reboot." + : + fi + fi + if test_fstab; then + fstab_uninstall_prompt + fi + + if test_nix_volume_mountd_installed; then + nix_volume_mountd_uninstall_prompt + fi +} + +setup_synthetic_conf() { + if test_nix_root_is_symlink; then + if ! test_synthetic_conf_symlinked; then + failure >&2 < $(readlink "$NIX_ROOT")). +Please remove it. If nix is in /etc/synthetic.conf, remove it and reboot. +EOF + fi + fi + if ! test_synthetic_conf_mountable; then + task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2 + # technically /etc/synthetic.d/nix is supported in Big Sur+ + # but handling both takes even more code... + _sudo "to add Nix to /etc/synthetic.conf" \ + /usr/bin/ex /etc/synthetic.conf <&2 + fi + create_synthetic_objects + if ! test_nix; then + failure >&2 <&2 + add_nix_vol_fstab_line "$volume_uuid" /usr/sbin/vifs + fi +} + +encrypt_volume() { + local volume_uuid="$1" + local volume_label="$2" + local password + # Note: mount/unmount are late additions to support the right order + # of operations for creating the volume and then baking its uuid into + # other artifacts; not as well-trod wrt to potential errors, race + # conditions, etc. + + /usr/sbin/diskutil mount "$volume_label" + + password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)" + _sudo "to add your Nix volume's password to Keychain" \ + /usr/bin/security -i </dev/null; do + : + done +} + +setup_volume() { + local use_special use_uuid profile_packages + task "Creating a Nix volume" >&2 + # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without. + + use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}" + + use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")} + + setup_fstab "$use_uuid" + + if should_encrypt_volume; then + encrypt_volume "$use_uuid" "$NIX_VOLUME_LABEL" + setup_volume_daemon "encrypted" "$use_uuid" + # TODO: might be able to save ~60ms by caching or setting + # this somewhere rather than re-checking here. + elif volume_encrypted "$use_special"; then + setup_volume_daemon "encrypted" "$use_uuid" + else + setup_volume_daemon "unencrypted" "$use_uuid" + fi + + await_volume + + # TODO: below is a vague kludge for now; I just don't know + # what if any safe action there is to take here. Also, the + # reminder isn't very helpful. + # I'm less sure where this belongs, but it also wants mounted, pre-install + if type -p nix-env; then + profile_packages="$(nix-env --query --installed)" + # TODO: can probably do below faster w/ read + # intentionally unquoted string to eat whitespace in wc output + # shellcheck disable=SC2046,SC2059 + if ! [ $(printf "$profile_packages" | /usr/bin/wc -l) = "0" ]; then + reminder <&2 + _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <&2 + + setup_darwin_volume + } + + main "$@" +fi diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index f6575ae2f..f8d6c5e8f 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,59 +3,99 @@ set -eu set -o pipefail -readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +# create by default; set 0 to DIY, use a symlink, etc. +readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default NIX_FIRST_BUILD_UID="301" NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" -dsclattr() { - /usr/bin/dscl . -read "$1" \ - | awk "/$2/ { print \$2 }" +# caution: may update times on / if not run as normal non-root user +read_only_root() { + # this touch command ~should~ always produce an error + # as of this change I confirmed /usr/bin/touch emits: + # "touch: /: Read-only file system" Catalina+ and Big Sur + # "touch: /: Permission denied" Mojave + # (not matching prefix for compat w/ coreutils touch in case using + # an explicit path causes problems; its prefix differs) + [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]] + + # Avoiding the slow semantic way to get this information (~330ms vs ~8ms) + # unless using touch causes problems. Just in case, that approach is: + # diskutil info -plist / | , i.e. + # diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" - } -poly_validate_assumptions() { - if [ "$(uname -s)" != "Darwin" ]; then - failure "This script is for use with macOS!" +if read_only_root && [ "$NIX_VOLUME_CREATE" = 1 ]; then + should_create_volume() { return 0; } +else + should_create_volume() { return 1; } +fi + +# shellcheck source=./create-darwin-volume.sh +. "$EXTRACTED_NIX_PATH/create-darwin-volume.sh" "no-main" + +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | /usr/bin/awk "/$2/ { print \$2 }" +} + +test_nix_daemon_installed() { + test -e "$NIX_DAEMON_DEST" +} + +poly_cure_artifacts() { + if should_create_volume; then + task "Fixing any leftover Nix volume state" + cat < /dev/null 2>&1 + /usr/sbin/dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 } poly_user_in_group_set() { @@ -151,3 +193,17 @@ poly_create_build_user() { /usr/bin/dscl . create "/Users/$username" \ UniqueID "${uid}" } + +poly_prepare_to_install() { + if should_create_volume; then + header "Preparing a Nix volume" + # intentional indent below to match task indent + cat < 1 )); then + header "Reminders" + for line in "${_reminders[@]}"; do + echo "$line" + if ! headless && [ "${#line}" = 0 ]; then + if read -r -p "Press enter/return to acknowledge."; then + printf $'\033[A\33[2K\r' + fi + fi + done + fi +} + +reminder() { + printf -v label "${BLUE}[ %d ]${ESC}" "$_remind_num" + _reminders+=("$label") + if [[ "$*" = "" ]]; then + while read -r line; do + _reminders+=("$line") + done + else + # this expands each arg to an array entry (and each entry will + # ultimately be a separate line in the output) + _reminders+=("$@") + fi + _reminders+=("") + ((_remind_num++)) +} + __sudo() { local expl="$1" local cmd="$2" @@ -221,18 +313,18 @@ _sudo() { local expl="$1" shift if ! headless; then - __sudo "$expl" "$*" + __sudo "$expl" "$*" >&2 fi sudo "$@" } -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { +readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +finish_cleanup() { rm -rf "$SCRATCH" } -function finish_fail { +finish_fail() { finish_cleanup failure < /dev/null >&2; then warning <&2 -elif [ "$(uname -s)" = "Linux" ]; then +if [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi -INSTALL_MODE=no-daemon -CREATE_DARWIN_VOLUME=0 +case "$(uname -s)" in + "Darwin") + INSTALL_MODE=daemon;; + *) + INSTALL_MODE=no-daemon;; +esac + +# space-separated string +ACTIONS= + # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) - INSTALL_MODE=daemon;; + INSTALL_MODE=daemon + ACTIONS="${ACTIONS}install " + ;; --no-daemon) - INSTALL_MODE=no-daemon;; + if [ "$(uname -s)" = "Darwin" ]; then + printf '\e[1;31mError: --no-daemon installs are no-longer supported on Darwin/macOS!\e[0m\n' >&2 + exit 1 + fi + INSTALL_MODE=no-daemon + # intentional tail space + ACTIONS="${ACTIONS}install " + ;; + # --uninstall) + # # intentional tail space + # ACTIONS="${ACTIONS}uninstall " + # ;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) @@ -69,13 +79,18 @@ while [ $# -gt 0 ]; do --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) - CREATE_DARWIN_VOLUME=1;; + { + echo "Warning: the flag --darwin-use-unencrypted-nix-store-volume" + echo " is no longer needed and will be removed in the future." + echo "" + } >&2;; --nix-extra-conf-file) - export NIX_EXTRA_CONF="$(cat $2)" + # shellcheck disable=SC2155 + export NIX_EXTRA_CONF="$(cat "$2")" shift;; *) - ( - echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" + { + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]" echo "Choose installation method." echo "" @@ -101,45 +116,16 @@ while [ $# -gt 0 ]; do if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from." fi - ) >&2 + } >&2 - # darwin and Catalina+ - if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then - ( - echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" - echo " store and mount it at /nix. This is the recommended way to create" - echo " /nix with a read-only / on macOS >=10.15." - echo " See: https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - fi exit;; esac shift done -if [ "$(uname -s)" = "Darwin" ]; then - if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then - printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' - "$self/create-darwin-volume.sh" - fi - - writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)" - if ! [ -e $dest ] && [ "$writable" = "false" ]; then - ( - echo "" - echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - exit 1 - fi -fi - if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" + exec "$self/install-multi-user" $ACTIONS # let ACTIONS split exit 0 fi @@ -194,6 +180,7 @@ if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then exit 1 fi +# shellcheck source=./nix-profile.sh.in . "$nix/etc/profile.d/nix.sh" if ! "$nix/bin/nix-env" -i "$nix"; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index fda5ef600..81c61b2a0 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -41,10 +41,8 @@ handle_network_proxy() { fi } -poly_validate_assumptions() { - if [ "$(uname -s)" != "Linux" ]; then - failure "This script is for use with Linux!" - fi +poly_cure_artifacts() { + : } poly_service_installed_check() { @@ -72,7 +70,7 @@ poly_service_setup_note() { EOF } -poly_extra_try_me_commands(){ +poly_extra_try_me_commands() { if [ -e /run/systemd/system ]; then : else @@ -81,19 +79,10 @@ poly_extra_try_me_commands(){ EOF fi } -poly_extra_setup_instructions(){ - if [ -e /run/systemd/system ]; then - : - else - cat < Date: Sun, 2 May 2021 05:30:50 +0100 Subject: [PATCH 009/555] Mark `__impureHostDeps` paths as optional Starting in macOS 11, the on-disk dylib bundles are no longer available, but nixpkgs needs to be able to keep compatibility with older versions that require `/usr/lib/libSystem.B.dylib` in `__impureHostDeps`. Allow it to keep backwards compatibility with these versions by marking these dependencies as optional. Fixes #4658. --- src/libstore/build/local-derivation-goal.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7c1402918..4a67b3f47 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -581,7 +581,9 @@ void LocalDerivationGoal::startBuilder() throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", worker.store.printStorePath(drvPath), i); - dirsInChroot[i] = i; + /* Allow files in __impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + dirsInChroot[i] = {i, true}; } #if __linux__ From e5951a6b2ffcb1b06c63bffb2549f00b12012c6d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 3 May 2021 01:12:23 +0200 Subject: [PATCH 010/555] Bump version number for `DerivedPath` changes I guess I misunderstood John's initial explanation about why wildcards for outputs are sent to older stores[1]. My `nix-daemon` from 2021-03-26 also has version 1.29, but misses the wildcard[2]. So bumping seems to be the right call. [1] https://github.com/NixOS/nix/pull/4759#issuecomment-830812464 [2] 255d145ba7ac907d1cba8d088da556b591627756 --- src/libstore/daemon.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/worker-protocol.hh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0be9d2c54..72b3e3e13 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -263,7 +263,7 @@ static void writeValidPathInfo( static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { reqs = worker_proto::read(store, from, Phantom> {}); } else { for (auto & s : readStrings(from)) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 761b4a087..d9b6e9488 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -672,7 +672,7 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { worker_proto::write(store, conn->to, reqs); } else { Strings ss; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 001ed25e3..fdd692cf0 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 29) +#define PROTOCOL_VERSION (1 << 8 | 30) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From dadfbce318c53ae0c53121deab6ab7e07c396164 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 4 May 2021 10:35:34 +0200 Subject: [PATCH 011/555] Fix the double-slash in the realisations path Make sure that we always access the realisations under `binaryCacheUrl/realisations` and not `binaryCacheUrl//realisations` Fix #4766 --- src/libstore/binary-cache-store.hh | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c2163166c..657be2fcf 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -34,7 +34,7 @@ private: protected: // The prefix under which realisation infos will be stored - const std::string realisationsPrefix = "/realisations"; + const std::string realisationsPrefix = "realisations"; BinaryCacheStore(const Params & params); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 964c4017e..f93111fce 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -93,7 +93,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + realisationsPrefix); + createDirs(binaryCacheDir + "/" + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); From 4f493faf8062d6c7e51277527a029d277b156330 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Tue, 4 May 2021 14:24:00 -0700 Subject: [PATCH 012/555] launchd: Use exec to avoid leaving the extra shell wrapper running Before: UID PID PPID C STIME TTY TIME CMD 0 1737 1 0 2:28PM ?? 0:00.00 /bin/sh -c /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon 0 1739 1737 0 2:28PM ?? 0:00.05 /nix/var/nix/profiles/default/bin/nix-daemon After: UID PID PPID C STIME TTY TIME CMD 0 1763 1 0 2:29PM ?? 0:00.05 /nix/var/nix/profiles/default/bin/nix-daemon Signed-off-by: Anders Kaseorg --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index c334639e2..f1b439840 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -19,7 +19,7 @@ /bin/sh -c - /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && exec /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log From 5f07f2ff2b964bec71be34cf6cf1265ce88b7490 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 5 May 2021 13:56:23 +0000 Subject: [PATCH 013/555] doc: builtins: use a definition list This looks a lot better (and is also more semantically meaningful). Since this list is generated in a Nix expression, I don't think using HTML here is going to be the thing that puts people off modifying this part of the documentation! --- doc/manual/generate-builtins.nix | 10 ++++++---- doc/manual/local.mk | 1 + doc/manual/src/expressions/builtins-prefix.md | 9 +++++---- doc/manual/src/expressions/builtins-suffix.md | 1 + 4 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 doc/manual/src/expressions/builtins-suffix.md diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 416a7fdba..92c7b1a31 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -6,9 +6,11 @@ builtins: concatStrings (map (name: let builtin = builtins.${name}; in - " - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args) - + " \n\n" - + concatStrings (map (s: " ${s}\n") (splitLines builtin.doc)) + "\n\n" + "
${name} " + + concatStringsSep " " (map (s: "${s}") builtin.args) + + "
" + + "
\n\n" + + builtin.doc + + "\n\n
" ) (attrNames builtins)) - diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 271529b38..e25157af8 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -64,6 +64,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md index c16b2805f..87127de2a 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/expressions/builtins-prefix.md @@ -9,7 +9,8 @@ scope. Instead, you can access them through the `builtins` built-in value, which is a set that contains all built-in functions and values. For instance, `derivation` is also available as `builtins.derivation`. - - `derivation` *attrs*; `builtins.derivation` *attrs*\ - - `derivation` is described in [its own section](derivations.md). - +
+
derivation attrs; + builtins.derivation attrs
+

derivation in described in + its own section.

diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/expressions/builtins-suffix.md new file mode 100644 index 000000000..a74db2857 --- /dev/null +++ b/doc/manual/src/expressions/builtins-suffix.md @@ -0,0 +1 @@ +
From bf485dcf460cd0df113c33f36c9f1f29d6e3a2d5 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 5 May 2021 21:00:08 +0200 Subject: [PATCH 014/555] Properly normalize the content-addressed paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that their timestamp are always normalized. Otherwise, strange − and non-deterministic − things might happen, like https://github.com/NixOS/nixpkgs/issues/121813 Fix #4775 --- src/libstore/build/local-derivation-goal.cc | 8 ++++---- tests/build.sh | 7 +++++++ tests/ca/build.sh | 8 ++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4a67b3f47..9c14c06b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2300,10 +2300,6 @@ void LocalDerivationGoal::registerOutputs() sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); } }; @@ -2452,6 +2448,10 @@ void LocalDerivationGoal::registerOutputs() }, }, output.output); + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); + /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ diff --git a/tests/build.sh b/tests/build.sh index aa54b88eb..ce9d6602c 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -10,3 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status ' (.drvPath | match(".*multiple-outputs-b.drv")) and (.outputs.out | match(".*multiple-outputs-b"))) ' + +testNormalization () { + clearStore + outPath=$(nix-build ./simple.nix) + test "$(stat -c %Y $outPath)" -eq 1 +} +testNormalization diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 35bf1dcf7..c8877f87f 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -59,9 +59,17 @@ testNixCommand () { nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link } +# Regression test for https://github.com/NixOS/nix/issues/4775 +testNormalization () { + clearStore + outPath=$(buildAttr rootCA 1) + test "$(stat -c %Y $outPath)" -eq 1 +} + # Disabled until we have it properly working # testRemoteCache clearStore +testNormalization testDeterministicCA clearStore testCutoff From 1e3a9033b7010c3b49c61c05ec93ba1ba6a61c97 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 5 May 2021 16:33:05 -0500 Subject: [PATCH 015/555] Use derivation output name from toDerivation This was previously done in https://github.com/NixOS/nix/pull/4515 but got clobbered away in https://github.com/NixOS/nix/pull/4594. -------------------------------------------------------------------------------- This fixes an issue where derivations with a primary output that is not "out" would fail with: $ nix profile install nixpkgs#sqlite error: opening directory '/nix/store/2a2ydlgyydly5czcc8lg12n6qqkfz863-sqlite-3.34.1-bin': No such file or directory This happens because while derivations produce every output when built, you might not have them if you didn't build the derivation yourself (for instance, the store path was fetch from a binary cache). This uses outputName provided from DerivationInfo which appears to match the first output of the derivation. --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 667904cd2..e511c4c3e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -426,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {drv.outputName}}); } } From b66234134fadc720d6c177d62e6ce4c9f4093a89 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 6 May 2021 16:45:09 +0200 Subject: [PATCH 016/555] Add a realisations disk cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similar to the nar-info disk cache (and using the same db). This makes rebuilds muuch faster. - This works regardless of the ca-derivations experimental feature. I could modify the logic to not touch the db if the flag isn’t there, but given that this is a trash-able local cache, it doesn’t seem to be really worth it. - We could unify the `NARs` and `Realisation` tables to only have one generic kv table. This is left as an exercise to the reader. - I didn’t update the cache db version number as the new schema just adds a new table to the previous one, so the db will be transparently migrated and is backwards-compatible. Fix #4746 --- src/libstore/binary-cache-store.cc | 29 +++++++- src/libstore/nar-info-disk-cache.cc | 100 +++++++++++++++++++++++++++- src/libstore/nar-info-disk-cache.hh | 10 +++ tests/ca/substitute.sh | 13 ++++ 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 09e1c254b..312630520 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] = + diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case (NarInfoDiskCache::oValid): + debug("Returning a cached realisation for %s", id.to_string()); + return *maybeCachedRealisation; + case (NarInfoDiskCache::oInvalid): + debug("Returning a cached missing realisation for %s", id.to_string()); + return {}; + case (NarInfoDiskCache::oUnknown): + break; + } + } + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return {Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + + if (diskCache) + diskCache->upsertRealisation( + getUri(), realisation); + + return {realisation}; } else { + if (diskCache) + diskCache->upsertAbsentRealisation(getUri(), id); return std::nullopt; } } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + if (diskCache) + diskCache->upsertRealisation(getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1d8d2d57e..84ce7972f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include +#include namespace nix { @@ -38,6 +39,16 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); +create table if not exists Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or empty if present is true + timestamp integer not null, + present integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + create table if not exists LastPurge ( dummy text primary key, value integer @@ -63,7 +74,9 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, + queryNAR, insertRealisation, insertMissingRealisation, + queryRealisation, purgeCache; std::map caches; }; @@ -98,6 +111,26 @@ public: state->queryNAR.create(state->db, "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + state->insertRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, content, timestamp, present) + values (?, ?, ?, ?, 1) + )"); + + state->insertMissingRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, timestamp, present) + values (?, ?, ?, 0) + )"); + + state->queryRealisation.create(state->db, + R"( + select present, content from Realisations + where cache = ? and outputId = ? and + ((present = 0 and timestamp > ?) or + (present = 1 and timestamp > ?)) + )"); + /* Periodically purge expired entries from the database. */ retrySQLite([&]() { auto now = time(0); @@ -212,6 +245,38 @@ public: }); } + std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) override + { + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryRealisation(state->queryRealisation.use() + (cache.id) + (id.to_string()) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryRealisation.next()) + return {oUnknown, 0}; + + if (queryRealisation.getInt(0) == 0) + return {oInvalid, 0}; + + auto realisation = + std::make_shared(Realisation::fromJSON( + nlohmann::json::parse(queryRealisation.getStr(1)), + "Local disk cache")); + + return {oValid, realisation}; + }); + } + void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) override @@ -251,6 +316,39 @@ public: } }); } + + void upsertRealisation( + const std::string & uri, + const Realisation & realisation) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + state->insertRealisation.use() + (cache.id) + (realisation.id.to_string()) + (realisation.toJSON().dump()) + (time(0)).exec(); + }); + + } + + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + state->insertMissingRealisation.use() + (cache.id) + (id.to_string()) + (time(0)).exec(); + }); + } }; ref getNarInfoDiskCache() diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 04de2c5eb..2dcaa76a4 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -2,6 +2,7 @@ #include "ref.hh" #include "nar-info.hh" +#include "realisation.hh" namespace nix { @@ -29,6 +30,15 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) = 0; + + virtual void upsertRealisation( + const std::string & uri, + const Realisation & realisation) = 0; + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) = 0; + virtual std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index 737c851a5..dfc4ea68e 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then echo "Realisations not rebuilt" exit 1 fi + +# Test the local realisation disk cache +buildDrvs --post-build-hook ../push-to-store.sh +clearStore +# Add the realisations of rootCA to the cachecache +clearCacheCache +export _NIX_FORCE_HTTP=1 +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +# Try rebuilding, but remove the realisations from the remote cache to force +# using the cachecache +clearStore +rm $REMOTE_STORE_DIR/realisations/* +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 From 2f63cc02de5e319708157f2b56f69581106e59b8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 8 May 2021 21:31:28 -0500 Subject: [PATCH 017/555] Create parent trusted list path before writing This makes sure that $HOME/.local/share/nix exists before we try to write to it. --- src/libexpr/flake/config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 63566131e..c8a5a319f 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -22,7 +22,9 @@ static TrustedList readTrustedList() static void writeTrustedList(const TrustedList & trustedList) { - writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); + auto path = trustedListPath(); + createDirs(dirOf(path)); + writeFile(path, nlohmann::json(trustedList).dump()); } void ConfigFile::apply() From 7f7f99f3505ba9bc3d330fe32d3e697a2ad45446 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 10 May 2021 11:47:00 +0200 Subject: [PATCH 018/555] Implement `builtins.floor` and `builtins.ceil` using the C library functions internally Closes #4782 Note: even though the type is internally called `NixFloat`, it's actually a `double`. --- src/libexpr/primops.cc | 40 +++++++++++++++++++++++++++++ tests/lang/eval-okay-floor-ceil.exp | 1 + tests/lang/eval-okay-floor-ceil.nix | 9 +++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/lang/eval-okay-floor-ceil.exp create mode 100644 tests/lang/eval-okay-floor-ceil.nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc4afaab8..3db9c6478 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,6 +21,8 @@ #include #include +#include + namespace nix { @@ -714,6 +716,44 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { .fun = prim_addErrorContext, }); +static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, ceil(value)); +} + +static RegisterPrimOp primop_ceil({ + .name = "__ceil", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next higher integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_ceil, +}); + +static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, floor(value)); +} + +static RegisterPrimOp primop_floor({ + .name = "__floor", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next lower integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_floor, +}); + /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/lang/eval-okay-floor-ceil.exp new file mode 100644 index 000000000..81f80420b --- /dev/null +++ b/tests/lang/eval-okay-floor-ceil.exp @@ -0,0 +1 @@ +"23;24;23;23" diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/lang/eval-okay-floor-ceil.nix new file mode 100644 index 000000000..d76a0d86e --- /dev/null +++ b/tests/lang/eval-okay-floor-ceil.nix @@ -0,0 +1,9 @@ +with import ./lib.nix; + +let + n1 = builtins.floor 23.5; + n2 = builtins.ceil 23.5; + n3 = builtins.floor 23; + n4 = builtins.ceil 23; +in + builtins.concatStringsSep ";" (map toString [ n1 n2 n3 n4 ]) From bec33858bbd053404bbf2c4f60e97cad13c365e0 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 10 May 2021 16:41:10 +0200 Subject: [PATCH 019/555] primops: math.h -> cmath Co-authored-by: Eelco Dolstra --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3db9c6478..e8569b654 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,7 +21,7 @@ #include #include -#include +#include namespace nix { From ab96c1ee504674ddbb0a09796af87d898e2bf753 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 10 May 2021 17:36:49 +0200 Subject: [PATCH 020/555] Remove useless parents I never remember the exact syntax of the `switch` statement --- src/libstore/binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 312630520..df401e6f4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -454,13 +454,13 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut auto [cacheOutcome, maybeCachedRealisation] = diskCache->lookupRealisation(getUri(), id); switch (cacheOutcome) { - case (NarInfoDiskCache::oValid): + case NarInfoDiskCache::oValid: debug("Returning a cached realisation for %s", id.to_string()); return *maybeCachedRealisation; - case (NarInfoDiskCache::oInvalid): + case NarInfoDiskCache::oInvalid: debug("Returning a cached missing realisation for %s", id.to_string()); return {}; - case (NarInfoDiskCache::oUnknown): + case NarInfoDiskCache::oUnknown: break; } } From d5d19582ef24af3754f7a2675f43d6828c3a8638 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 10 May 2021 17:45:53 +0200 Subject: [PATCH 021/555] Simplify the realisations disk cache --- src/libstore/nar-info-disk-cache.cc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 84ce7972f..9dd81ddfb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -42,9 +42,8 @@ create table if not exists NARs ( create table if not exists Realisations ( cache integer not null, outputId text not null, - content blob, -- Json serialisation of the realisation, or empty if present is true + content blob, -- Json serialisation of the realisation, or null if the realisation is absent timestamp integer not null, - present integer not null, primary key (cache, outputId), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -113,22 +112,22 @@ public: state->insertRealisation.create(state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp, present) - values (?, ?, ?, ?, 1) + insert or replace into Realisations(cache, outputId, content, timestamp) + values (?, ?, ?, ?) )"); state->insertMissingRealisation.create(state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp, present) - values (?, ?, ?, 0) + insert or replace into Realisations(cache, outputId, timestamp) + values (?, ?, ?) )"); state->queryRealisation.create(state->db, R"( - select present, content from Realisations + select content from Realisations where cache = ? and outputId = ? and - ((present = 0 and timestamp > ?) or - (present = 1 and timestamp > ?)) + ((content is null and timestamp > ?) or + (content is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -265,12 +264,12 @@ public: if (!queryRealisation.next()) return {oUnknown, 0}; - if (queryRealisation.getInt(0) == 0) + if (queryRealisation.isNull(0)) return {oInvalid, 0}; auto realisation = std::make_shared(Realisation::fromJSON( - nlohmann::json::parse(queryRealisation.getStr(1)), + nlohmann::json::parse(queryRealisation.getStr(0)), "Local disk cache")); return {oValid, realisation}; From a3553cdae8270d5c9fd198a29415d17025f15980 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Tue, 11 May 2021 14:29:26 +0200 Subject: [PATCH 022/555] doc/nix-shell: Remove outdated bashrc information This behaviour was removed in 65f6d5db6f5ef2724a3dc03e1773c510123287f1. --- doc/manual/src/command-ref/nix-shell.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index dcd7cc70c..72f6730f1 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -78,9 +78,7 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. Note that (depending on your Bash - installation) `/etc/bashrc` is still sourced, so any variables set - there will affect the interactive shell. + retained. - `--packages` / `-p` *packages*…\ Set up an environment in which the specified packages are present. From d169eb2a1badc5ea32d21bc757d767927ba813c5 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Tue, 11 May 2021 06:43:44 -0700 Subject: [PATCH 023/555] install: Fix hash variable for Darwin.arm64 (#4769) When commit 233b61d3d6482544c35b9d340240bf3260acff13 (#4224) renamed @binaryTarball_${system}@ to @tarballHash_${system}@, it updated this reference for every platform except Darwin.arm64. Signed-off-by: Anders Kaseorg --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 7d25f7bd7..f660e3807 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -51,7 +51,7 @@ case "$(uname -s).$(uname -m)" in oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" fi - hash=@binaryTarball_x86_64-darwin@ + hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ # eventually maybe: aarch64-darwin system=x86_64-darwin From 4029f4b05bfffcf6c5cbbfae1bfb9416c070b81e Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 12 May 2021 13:25:09 +0200 Subject: [PATCH 024/555] doc: mention Priority for substituters --- src/libstore/globals.hh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 6f8749254..dd570cd63 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -617,8 +617,10 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. The default - is `https://cache.nixos.org`. + A list of URLs of substituters, separated by whitespace. Substituters + are tried based on their Priority value, which each substituter can set + independently. Lower value means higher priority. + The default is `https://cache.nixos.org`, with a Priority of 40. )", {"binary-caches"}}; From ec613603ba324bf12f8f554d74fb1a02c6e9b472 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 12 May 2021 16:19:51 +0200 Subject: [PATCH 025/555] DerivedPathWithHints -> BuiltPath Just a renaming for now --- src/libcmd/command.cc | 6 +++--- src/libcmd/command.hh | 4 ++-- src/libcmd/installables.cc | 36 ++++++++++++++++++------------------ src/libcmd/installables.hh | 6 +++--- src/libstore/derived-path.cc | 6 +++--- src/libstore/derived-path.hh | 20 ++++++++++---------- src/nix/build.cc | 4 ++-- src/nix/develop.cc | 6 +++--- src/nix/log.cc | 6 +++--- src/nix/profile.cc | 4 ++-- 10 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 9da470c15..25e4873e8 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -162,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) +void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { result.push_back(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 9e18c6e51..952279f7b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -DerivedPathsWithHints build(ref store, Realise mode, +BuiltPaths build(ref store, Realise mode, std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, @@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const DerivedPathsWithHints & buildables); + void updateProfile(const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 06ef4c669..36d7ecc39 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -285,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -DerivedPathWithHints Installable::toDerivedPathWithHints() +BuiltPath Installable::toBuiltPath() { - auto buildables = toDerivedPathsWithHints(); + auto buildables = toBuiltPaths(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -321,7 +321,7 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - DerivedPathsWithHints toDerivedPathsWithHints() override + BuiltPaths toBuiltPaths() override { if (storePath.isDerivation()) { std::map> outputs; @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - DerivedPathWithHints::Built { + BuiltPath::Built { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - DerivedPathWithHints::Opaque { + BuiltPath::Opaque { .path = storePath, } }; @@ -349,9 +349,9 @@ struct InstallableStorePath : Installable } }; -DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() +BuiltPaths InstallableValue::toBuiltPaths() { - DerivedPathsWithHints res; + BuiltPaths res; std::map>> drvsToOutputs; @@ -364,7 +364,7 @@ DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() } for (auto & i : drvsToOutputs) - res.push_back(DerivedPathWithHints::Built { i.first, i.second }); + res.push_back(BuiltPath::Built { i.first, i.second }); return res; } @@ -675,23 +675,23 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -DerivedPathsWithHints build(ref store, Realise mode, +BuiltPaths build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - DerivedPathsWithHints buildables; + BuiltPaths buildables; std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toDerivedPathsWithHints()) { + for (auto & b : i->toBuiltPaths()) { std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { pathsToBuild.push_back(bo); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); @@ -721,10 +721,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { res.insert(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -789,9 +789,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toDerivedPathsWithHints()) + for (auto & b : i->toBuiltPaths()) std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -800,7 +800,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { drvPaths.insert(bfd.drvPath); }, }, b.raw()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 403403c07..7d79efaad 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -29,9 +29,9 @@ struct Installable virtual std::string what() = 0; - virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0; + virtual BuiltPaths toBuiltPaths() = 0; - DerivedPathWithHints toDerivedPathWithHints(); + BuiltPath toBuiltPath(); App toApp(EvalState & state); @@ -74,7 +74,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - DerivedPathsWithHints toDerivedPathsWithHints() override; + BuiltPaths toBuiltPaths() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 13833c58e..d38613bd3 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -11,7 +11,7 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } -nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { +nlohmann::json BuiltPath::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -20,9 +20,9 @@ nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { return res; } -nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store) { +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) { auto res = nlohmann::json::array(); - for (const DerivedPathWithHints & buildable : buildables) { + for (const BuiltPath & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); }, buildable.raw()); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a2fe59de..72304f4fa 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -79,19 +79,19 @@ struct DerivedPath : _DerivedPathRaw { /** * A built derived path with hints in the form of optional concrete output paths. * - * See 'DerivedPathWithHints' for more an explanation. + * See 'BuiltPath' for more an explanation. */ -struct DerivedPathWithHintsBuilt { +struct BuiltPathBuilt { StorePath drvPath; std::map> outputs; nlohmann::json toJSON(ref store) const; - static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); + static BuiltPathBuilt parse(const Store & store, std::string_view); }; -using _DerivedPathWithHintsRaw = std::variant< +using _BuiltPathRaw = std::variant< DerivedPath::Opaque, - DerivedPathWithHintsBuilt + BuiltPathBuilt >; /** @@ -109,12 +109,12 @@ using _DerivedPathWithHintsRaw = std::variant< * paths. */ // FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. -struct DerivedPathWithHints : _DerivedPathWithHintsRaw { - using Raw = _DerivedPathWithHintsRaw; +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; - using Built = DerivedPathWithHintsBuilt; + using Built = BuiltPathBuilt; inline const Raw & raw() const { return static_cast(*this); @@ -122,8 +122,8 @@ struct DerivedPathWithHints : _DerivedPathWithHintsRaw { }; -typedef std::vector DerivedPathsWithHints; +typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); } diff --git a/src/nix/build.cc b/src/nix/build.cc index 226c551fa..d924fe553 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -63,12 +63,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 498a7b45c..3762e5bcc 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -265,7 +265,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toDerivedPathWithHints(); + auto buildable = installable->toBuiltPath(); auto doRedirect = [&](const StorePath & path) { auto from = store->printStorePath(path); @@ -277,10 +277,10 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const DerivedPathWithHints::Opaque & bo) { + [&](const BuiltPath::Opaque & bo) { doRedirect(bo.path); }, - [&](const DerivedPathWithHints::Built & bfd) { + [&](const BuiltPath::Built & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, diff --git a/src/nix/log.cc b/src/nix/log.cc index 638bb5073..d87fda0b8 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toDerivedPathWithHints(); + auto b = installable->toBuiltPath(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b.raw()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index e511c4c3e..511771f89 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -259,11 +259,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](BuiltPath::Opaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](BuiltPath::Built bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? From 9a143358454ba8268b72846ae3a5ef65f814e660 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 12 May 2021 12:15:32 -0500 Subject: [PATCH 026/555] Relock wait for build slot goals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we don’t have enough free job slots to run a goal, we put it in the waitForBuildSlot list & unlock its output locks. This will continue from where we left off (tryLocalBuild). However, we need the locks to get reacquired when/if the goal ever restarts. So, we need to send it back through tryToBuild to get reqacquire those locks. I think this bug was introduced in https://github.com/NixOS/nix/pull/4570. It leads to some builds starting without proper locks. --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c14c06b4..3de12c6ff 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -153,6 +153,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { + state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; From 8c7e043de2f673bc355d83f1e873baa93f30be62 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 12 May 2021 21:40:28 -0500 Subject: [PATCH 027/555] Fix tokenize output names in drv This should fix the issue described in https://discourse.nixos.org/t/derivation-does-not-have-wanted-outputs-dev-out/12905. Specifically, we get an error of error: derivation '/nix/store/_.drv' does not have wanted outputs 'dev,out' when a path like /nix/store/_.drv!dev,out is sent to the daemon. --- src/libstore/derived-path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 13833c58e..9cfbf50d9 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -62,7 +62,7 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi auto outputsS = s.substr(n + 1); std::set outputs; if (outputsS != "*") - outputs = tokenizeString>(outputsS); + outputs = tokenizeString>(outputsS, ","); return {drvPath, outputs}; } From de9e43c2ea8ac8752d42660c17a7b683682ef38b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 May 2021 14:03:53 +0200 Subject: [PATCH 028/555] Update nix-fallback-paths.nix only --- maintainers/upload-release.pl | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 6f3882a12..c2933300f 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -133,20 +133,8 @@ for my $fn (glob "$tmpDir/*") { exit if $version =~ /pre/; -# Update Nixpkgs in a very hacky way. +# Update nix-fallback-paths.nix. system("cd $nixpkgsDir && git pull") == 0 or die; -my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName; -my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash; -print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n"; - -my $fn = "$nixpkgsDir/pkgs/tools/package-management/nix/default.nix"; -my $oldFile = read_file($fn); -$oldFile =~ s/$oldName/"$releaseName"/g; -$oldFile =~ s/$oldHash/"$tarballHash"/g; -write_file($fn, $oldFile); - -$oldName =~ s/nix-//g; -$oldName =~ s/"//g; sub getStorePath { my ($jobName) = @_; @@ -167,7 +155,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . "}\n"); -system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; +system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; # Update the "latest" symlink. $channelsBucket->add_key( From 559a504da7d4289badfb4ebf7906f80fbf50bedf Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 17 May 2021 00:15:30 +0100 Subject: [PATCH 029/555] sandbox: allow SystemVersionCompat.plist on Darwin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For whatever reason, many programs trying to access SystemVersion.plist also open SystemVersionCompat.plist; this includes Python code and coreutils’ `cat(1)` (but not the native macOS `/bin/cat`). Illustratory `dtruss(1m)` output: open("/System/Library/CoreServices/SystemVersion.plist\0", 0x0, 0x0) = 3 0 open("/System/Library/CoreServices/SystemVersionCompat.plist\0", 0x0, 0x0) = 4 0 I assume this is a Big Sur change relating to the 10.16.x/11.x version compatibility divide and that it’s something along the lines of a hook inside libSystem. Fixes a lot of sandboxed package builds under Big Sur. --- src/libstore/sandbox-defaults.sb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 351037822..2bb1ea130 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -32,7 +32,9 @@ (literal "/tmp") (subpath TMPDIR)) ; Some packages like to read the system version. -(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) ; Without this line clang cannot write to /dev/null, breaking some configure tests. (allow file-read-metadata (literal "/dev")) From 21050846457f356346204dd52fb7a6d49f710688 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 17 May 2021 08:45:08 +0200 Subject: [PATCH 030/555] Enfore the use of properly built paths in libcmd Replace `DerivedPathWithHints` by a new `BuiltPath` type that serves as a proof that the corresponding path has been built. --- src/libcmd/command.cc | 39 ++++----- src/libcmd/command.hh | 14 ++-- src/libcmd/installables.cc | 151 ++++++++++++++++++----------------- src/libcmd/installables.hh | 6 +- src/libstore/derived-path.cc | 43 +++++++++- src/libstore/derived-path.hh | 22 ++--- src/nix/copy.cc | 20 +++-- src/nix/develop.cc | 17 +--- src/nix/log.cc | 6 +- src/nix/realisation.cc | 15 +++- 10 files changed, 189 insertions(+), 144 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 25e4873e8..569c4b9e4 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,7 +54,7 @@ void StoreCommand::run() run(getStore()); } -RealisedPathsCommand::RealisedPathsCommand(bool recursive) +BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { if (recursive) @@ -81,39 +81,45 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive) }); } -void RealisedPathsCommand::run(ref store) +void BuiltPathsCommand::run(ref store) { - std::vector paths; + BuiltPaths paths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(p); + paths.push_back(BuiltPath::Opaque{p}); } else { - auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables); + paths = toBuiltPaths(store, realiseMode, operateOn, installables); if (recursive) { - auto roots = std::move(pathSet); - pathSet = {}; - RealisedPath::closure(*store, roots, pathSet); + // XXX: This only computes the store path closure, ignoring + // intermediate realisations + StorePathSet pathsRoots, pathsClosure; + for (auto & root: paths) { + auto rootFromThis = root.outPaths(); + pathsRoots.insert(rootFromThis.begin(), rootFromThis.end()); + } + store->computeFSClosure(pathsRoots, pathsClosure); + for (auto & path : pathsClosure) + paths.push_back(BuiltPath::Opaque{path}); } - for (auto & path : pathSet) - paths.push_back(path); } run(store, std::move(paths)); } StorePathsCommand::StorePathsCommand(bool recursive) - : RealisedPathsCommand(recursive) + : BuiltPathsCommand(recursive) { } -void StorePathsCommand::run(ref store, std::vector paths) +void StorePathsCommand::run(ref store, BuiltPaths paths) { StorePaths storePaths; - for (auto & p : paths) - storePaths.push_back(p.path()); + for (auto& builtPath : paths) + for (auto& p : builtPath.outPaths()) + storePaths.push_back(p); run(store, std::move(storePaths)); } @@ -175,10 +181,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) }, [&](BuiltPath::Built bfd) { for (auto & output : bfd.outputs) { - /* Output path should be known because we just tried to - build it. */ - assert(output.second); - result.push_back(*output.second); + result.push_back(output.second); } }, }, buildable.raw()); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 952279f7b..35b3a384b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -143,7 +143,7 @@ private: }; /* A command that operates on zero or more store paths. */ -struct RealisedPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : public InstallablesCommand { private: @@ -156,26 +156,26 @@ protected: public: - RealisedPathsCommand(bool recursive = false); + BuiltPathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, std::vector paths) = 0; + virtual void run(ref store, BuiltPaths paths) = 0; void run(ref store) override; bool useDefaultInstallables() override { return !all; } }; -struct StorePathsCommand : public RealisedPathsCommand +struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; virtual void run(ref store, std::vector storePaths) = 0; - void run(ref store, std::vector paths) override; + void run(ref store, BuiltPaths paths) override; }; /* A command that operates on exactly one store path. */ @@ -231,7 +231,7 @@ std::set toDerivations(ref store, std::vector> installables, bool useDeriver = false); -std::set toRealisedPaths( +BuiltPaths toBuiltPaths( ref store, Realise mode, OperateOn operateOn, diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 36d7ecc39..fe52912cf 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -285,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -BuiltPath Installable::toBuiltPath() +DerivedPath Installable::toDerivedPath() { - auto buildables = toBuiltPaths(); + auto buildables = toDerivedPaths(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -321,22 +321,19 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - BuiltPaths toBuiltPaths() override + DerivedPaths toDerivedPaths() override { if (storePath.isDerivation()) { - std::map> outputs; auto drv = store->readDerivation(storePath); - for (auto & [name, output] : drv.outputsAndOptPaths(*store)) - outputs.emplace(name, output.second); return { - BuiltPath::Built { + DerivedPath::Built { .drvPath = storePath, - .outputs = std::move(outputs) + .outputs = drv.outputNames(), } }; } else { return { - BuiltPath::Opaque { + DerivedPath::Opaque { .path = storePath, } }; @@ -349,22 +346,22 @@ struct InstallableStorePath : Installable } }; -BuiltPaths InstallableValue::toBuiltPaths() +DerivedPaths InstallableValue::toDerivedPaths() { - BuiltPaths res; + DerivedPaths res; - std::map>> drvsToOutputs; + std::map> drvsToOutputs; // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { auto outputName = drv.outputName; if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); - drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); + drvsToOutputs[drv.drvPath].insert(outputName); } for (auto & i : drvsToOutputs) - res.push_back(BuiltPath::Built { i.first, i.second }); + res.push_back(DerivedPath::Built { i.first, i.second }); return res; } @@ -675,32 +672,67 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } +BuiltPaths getBuiltPaths(ref store, DerivedPaths hopefullyBuiltPaths) +{ + BuiltPaths res; + for (auto& b : hopefullyBuiltPaths) + std::visit( + overloaded{ + [&](DerivedPath::Opaque bo) { + res.push_back(BuiltPath::Opaque{bo.path}); + }, + [&](DerivedPath::Built bfd) { + OutputPathMap outputs; + auto drv = store->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*store, drv); + auto drvOutputs = drv.outputsAndOptPaths(*store); + for (auto& output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output " + "named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled( + "ca-derivations")) { + auto outputId = + DrvOutput{outputHashes.at(output), output}; + auto realisation = + store->queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addresed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign( + output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); + } + } + res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); + }, + }, + b.raw()); + + return res; +} + BuiltPaths build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - BuiltPaths buildables; - std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toBuiltPaths()) { - std::visit(overloaded { - [&](BuiltPath::Opaque bo) { - pathsToBuild.push_back(bo); - }, - [&](BuiltPath::Built bfd) { - StringSet outputNames; - for (auto & output : bfd.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back( - DerivedPath::Built{bfd.drvPath, outputNames}); - }, - }, b.raw()); - buildables.push_back(std::move(b)); - } + auto b = i->toDerivedPaths(); + pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); } if (mode == Realise::Nothing) @@ -708,57 +740,26 @@ BuiltPaths build(ref store, Realise mode, else if (mode == Realise::Outputs) store->buildPaths(pathsToBuild, bMode); - return buildables; + return getBuiltPaths(store, pathsToBuild); } -std::set toRealisedPaths( +BuiltPaths toBuiltPaths( ref store, Realise mode, OperateOn operateOn, std::vector> installables) { - std::set res; if (operateOn == OperateOn::Output) { - for (auto & b : build(store, mode, installables)) - std::visit(overloaded { - [&](BuiltPath::Opaque bo) { - res.insert(bo.path); - }, - [&](BuiltPath::Built bfd) { - auto drv = store->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*store, drv); - for (auto & output : bfd.outputs) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (!outputHashes.count(output.first)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), - output.first); - auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string()); - res.insert(RealisedPath{*realisation}); - } - else { - // If ca-derivations isn't enabled, behave as if - // all the paths are opaque to keep the default - // behavior - assert(output.second); - res.insert(*output.second); - } - } - }, - }, b.raw()); + return build(store, mode, installables); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; - auto drvPaths = toDerivations(store, installables, true); - res.insert(drvPaths.begin(), drvPaths.end()); + BuiltPaths res; + for (auto & drvPath : toDerivations(store, installables, true)) + res.push_back(BuiltPath::Opaque{drvPath}); + return res; } - - return res; } StorePathSet toStorePaths(ref store, @@ -766,8 +767,10 @@ StorePathSet toStorePaths(ref store, std::vector> installables) { StorePathSet outPaths; - for (auto & path : toRealisedPaths(store, mode, operateOn, installables)) - outPaths.insert(path.path()); + for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(thisOutPaths.begin(), thisOutPaths.end()); + } return outPaths; } @@ -789,9 +792,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuiltPaths()) + for (auto & b : i->toDerivedPaths()) std::visit(overloaded { - [&](BuiltPath::Opaque bo) { + [&](DerivedPath::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -800,7 +803,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](BuiltPath::Built bfd) { + [&](DerivedPath::Built bfd) { drvPaths.insert(bfd.drvPath); }, }, b.raw()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 7d79efaad..16bc4ae68 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -29,9 +29,9 @@ struct Installable virtual std::string what() = 0; - virtual BuiltPaths toBuiltPaths() = 0; + virtual DerivedPaths toDerivedPaths() = 0; - BuiltPath toBuiltPath(); + DerivedPath toDerivedPath(); App toApp(EvalState & state); @@ -74,7 +74,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - BuiltPaths toBuiltPaths() override; + DerivedPaths toDerivedPaths() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index d38613bd3..f188ef3c1 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -15,11 +15,26 @@ nlohmann::json BuiltPath::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; + res["outputs"][output] = store->printStorePath(path); } return res; } +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](BuiltPath::Opaque p) { return StorePathSet{p.path}; }, + [](BuiltPath::Built b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) { auto res = nlohmann::json::array(); for (const BuiltPath & buildable : buildables) { @@ -74,4 +89,30 @@ DerivedPath DerivedPath::parse(const Store & store, std::string_view s) : (DerivedPath) DerivedPath::Built::parse(store, s); } +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](BuiltPath::Opaque p) { res.insert(p.path); }, + [&](BuiltPath::Built p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath)); + for (auto& [outputName, outputPath] : p.outputs) { + if (settings.isExperimentalFeatureEnabled( + "ca-derivations")) { + auto thisRealisation = store.queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName}); + assert(thisRealisation); // We’ve built it, so we must h + // ve the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 72304f4fa..9d6ace069 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "realisation.hh" #include @@ -83,7 +84,7 @@ struct DerivedPath : _DerivedPathRaw { */ struct BuiltPathBuilt { StorePath drvPath; - std::map> outputs; + std::map outputs; nlohmann::json toJSON(ref store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); @@ -95,20 +96,9 @@ using _BuiltPathRaw = std::variant< >; /** - * A derived path with hints in the form of optional concrete output paths in the built case. - * - * This type is currently just used by the CLI. The paths are filled in - * during evaluation for derivations that know what paths they will - * produce in advanced, i.e. input-addressed or fixed-output content - * addressed derivations. - * - * That isn't very good, because it puts floating content-addressed - * derivations "at a disadvantage". It would be better to never rely on - * the output path of unbuilt derivations, and exclusively use the - * realizations types to work with built derivations' concrete output - * paths. + * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * output path(s). */ -// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. struct BuiltPath : _BuiltPathRaw { using Raw = _BuiltPathRaw; using Raw::Raw; @@ -120,8 +110,12 @@ struct BuiltPath : _BuiltPathRaw { return static_cast(*this); } + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + }; +typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f59f7c76b..674cce4b4 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : RealisedPathsCommand +struct CmdCopy : BuiltPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : RealisedPathsCommand SubstituteFlag substitute = NoSubstitute; - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; CmdCopy() - : RealisedPathsCommand(true) + : BuiltPathsCommand(true) { addFlag({ .longName = "from", @@ -75,16 +75,22 @@ struct CmdCopy : RealisedPathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - RealisedPathsCommand::run(store); + BuiltPathsCommand::run(store); } - void run(ref srcStore, std::vector paths) override + void run(ref srcStore, BuiltPaths paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + RealisedPath::Set stuffToCopy; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*srcStore); + stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); + } + copyPaths( - srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), - NoRepair, checkSigs, substitute); + srcStore, dstStore, stuffToCopy, NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3762e5bcc..2b64d9a31 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -265,9 +265,9 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toBuiltPath(); - auto doRedirect = [&](const StorePath & path) - { + auto builtPaths = toStorePaths( + store, Realise::Nothing, OperateOn::Output, {installable}); + for (auto & path: builtPaths) { auto from = store->printStorePath(path); if (script.find(from) == std::string::npos) warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); @@ -275,16 +275,7 @@ struct Common : InstallableCommand, MixProfile printInfo("redirecting '%s' to '%s'", from, dir); rewrites.insert({from, dir}); } - }; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - doRedirect(bo.path); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & [outputName, path] : bfd.outputs) - if (path) doRedirect(*path); - }, - }, buildable.raw()); + } } return rewriteStrings(script, rewrites); diff --git a/src/nix/log.cc b/src/nix/log.cc index d87fda0b8..962c47525 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuiltPath(); + auto b = installable->toDerivedPath(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](BuiltPath::Opaque bo) { + [&](DerivedPath::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](BuiltPath::Built bfd) { + [&](DerivedPath::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b.raw()); diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 9ee9ccb91..d59e594df 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -28,7 +28,7 @@ struct CmdRealisation : virtual NixMultiCommand static auto rCmdRealisation = registerCommand("realisation"); -struct CmdRealisationInfo : RealisedPathsCommand, MixJSON +struct CmdRealisationInfo : BuiltPathsCommand, MixJSON { std::string description() override { @@ -44,12 +44,19 @@ struct CmdRealisationInfo : RealisedPathsCommand, MixJSON Category category() override { return catSecondary; } - void run(ref store, std::vector paths) override + void run(ref store, BuiltPaths paths) override { settings.requireExperimentalFeature("ca-derivations"); + RealisedPath::Set realisations; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*store); + realisations.insert(theseRealisations.begin(), theseRealisations.end()); + } + if (json) { nlohmann::json res = nlohmann::json::array(); - for (auto & path : paths) { + for (auto & path : realisations) { nlohmann::json currentPath; if (auto realisation = std::get_if(&path.raw)) currentPath = realisation->toJSON(); @@ -61,7 +68,7 @@ struct CmdRealisationInfo : RealisedPathsCommand, MixJSON std::cout << res.dump(); } else { - for (auto & path : paths) { + for (auto & path : realisations) { if (auto realisation = std::get_if(&path.raw)) { std::cout << realisation->id.to_string() << " " << From f0c0052d4b16d13d2a826a2839b680b729651179 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 17 May 2021 09:48:51 +0200 Subject: [PATCH 031/555] Resolve the program path in `nix run` Fix #4768 --- src/nix/app.cc | 96 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 17 deletions(-) diff --git a/src/nix/app.cc b/src/nix/app.cc index cf147c631..1e30deb34 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -3,9 +3,61 @@ #include "eval-inline.hh" #include "eval-cache.hh" #include "names.hh" +#include "command.hh" namespace nix { +struct InstallableDerivedPath : Installable +{ + ref store; + const DerivedPath derivedPath; + + InstallableDerivedPath(ref store, const DerivedPath & derivedPath) + : store(store) + , derivedPath(derivedPath) + { + } + + + std::string what() override { return derivedPath.to_string(*store); } + + DerivedPaths toDerivedPaths() override + { + return {derivedPath}; + } + + std::optional getStorePath() override + { + return std::nullopt; + } +}; + +/** + * Return the rewrites that are needed to resolve a string whose context is + * included in `dependencies` + */ +StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +{ + StringPairs res; + for (auto & dep : dependencies) + if (auto drvDep = std::get_if(&dep)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + downstreamPlaceholder(store, drvDep->drvPath, outputName), + store.printStorePath(outputPath) + ); + return res; +} + +/** + * Resolve the given string assuming the given context + */ +std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +{ + auto rewrites = resolveRewrites(store, dependencies); + return rewriteStrings(toResolve, rewrites); +} + App Installable::toApp(EvalState & state) { auto [cursor, attrPath] = getCursor(state); @@ -18,19 +70,21 @@ App Installable::toApp(EvalState & state) throw Error("app program '%s' is not in the Nix store", program); }; + std::vector> context; + std::string unresolvedProgram; + + if (type == "app") { - auto [program, context] = cursor->getAttr("program")->getStringWithContext(); + auto [program, context_] = cursor->getAttr("program")->getStringWithContext(); + unresolvedProgram = program; - checkProgram(program); - - std::vector context2; - for (auto & [path, name] : context) - context2.push_back({state.store->parseStorePath(path), {name}}); - - return App { - .context = std::move(context2), - .program = program, - }; + for (auto & [path, name] : context_) + context.push_back(std::make_shared( + state.store, + DerivedPathBuilt{ + .drvPath = state.store->parseStorePath(path), + .outputs = {name}, + })); } else if (type == "derivation") { @@ -44,16 +98,24 @@ App Installable::toApp(EvalState & state) aMainProgram ? aMainProgram->getString() : DrvName(name).name; - auto program = outPath + "/bin/" + mainProgram; - checkProgram(program); - return App { - .context = { { drvPath, {outputName} } }, - .program = program, - }; + unresolvedProgram = outPath + "/bin/" + mainProgram; + context = {std::make_shared( + state.store, + DerivedPathBuilt{ + .drvPath = drvPath, + .outputs = {outputName}, + })}; } else throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); + + auto builtContext = build(state.store, Realise::Outputs, context); + auto program = resolveString(*state.store, unresolvedProgram, builtContext); + checkProgram(program); + return App { + .program = program, + }; } } From f46adb783c939c61a087cdfd8af964f4ad6f9816 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 11 May 2021 10:52:17 +0200 Subject: [PATCH 032/555] Add a test for `nix run` with CA derivations --- .gitignore | 1 + tests/ca/config.nix.in | 1 + tests/ca/content-addressed.nix | 34 ++++++++++++++++++++-------------- tests/ca/flake.nix | 3 +++ tests/ca/nix-run.sh | 9 +++++++++ tests/local.mk | 5 +++-- 6 files changed, 37 insertions(+), 16 deletions(-) create mode 120000 tests/ca/config.nix.in create mode 100644 tests/ca/flake.nix create mode 100755 tests/ca/nix-run.sh diff --git a/.gitignore b/.gitignore index 37aada307..2e14561fe 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ perl/Makefile.config /tests/shell /tests/shell.drv /tests/config.nix +/tests/ca/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/tests/ca/config.nix.in b/tests/ca/config.nix.in new file mode 120000 index 000000000..af24ddb30 --- /dev/null +++ b/tests/ca/config.nix.in @@ -0,0 +1 @@ +../config.nix.in \ No newline at end of file diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index e5b1c4de3..d328fc92c 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -1,4 +1,11 @@ -with import ../config.nix; +with import ./config.nix; + +let mkCADerivation = args: mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; +} // args); +in { seed ? 0 }: # A simple content-addressed derivation. @@ -14,7 +21,7 @@ rec { echo "Hello World" > $out/hello ''; }; - rootCA = mkDerivation { + rootCA = mkCADerivation { name = "rootCA"; outputs = [ "out" "dev" "foo"]; buildCommand = '' @@ -27,11 +34,8 @@ rec { ln -s $out $dev ln -s $out $foo ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; }; - dependentCA = mkDerivation { + dependentCA = mkCADerivation { name = "dependent"; buildCommand = '' echo "building a dependent derivation" @@ -39,20 +43,14 @@ rec { cat ${rootCA}/self/dep echo ${rootCA}/self/dep > $out/dep ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; }; - transitivelyDependentCA = mkDerivation { + transitivelyDependentCA = mkCADerivation { name = "transitively-dependent"; buildCommand = '' echo "building transitively-dependent" cat ${dependentCA}/dep echo ${dependentCA} > $out ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; }; dependentNonCA = mkDerivation { name = "dependent-non-ca"; @@ -72,6 +70,14 @@ rec { cat ${dependentCA}/dep echo foo > $out ''; - + }; + runnable = mkCADerivation rec { + name = "runnable-thing"; + buildCommand = '' + mkdir -p $out/bin + echo ${rootCA} # Just to make it depend on it + echo "" > $out/bin/${name} + chmod +x $out/bin/${name} + ''; }; } diff --git a/tests/ca/flake.nix b/tests/ca/flake.nix new file mode 100644 index 000000000..332c92a67 --- /dev/null +++ b/tests/ca/flake.nix @@ -0,0 +1,3 @@ +{ + outputs = { self }: import ./content-addressed.nix {}; +} diff --git a/tests/ca/nix-run.sh b/tests/ca/nix-run.sh new file mode 100755 index 000000000..81402af10 --- /dev/null +++ b/tests/ca/nix-run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf + +FLAKE_PATH=path:$PWD + +nix run --no-write-lock-file $FLAKE_PATH#runnable diff --git a/tests/local.mk b/tests/local.mk index e2c94dde6..542be6b7e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -46,6 +46,7 @@ nix_tests = \ ca/build.sh \ ca/substitute.sh \ ca/signatures.sh \ + ca/nix-run.sh \ ca/nix-copy.sh # parallel.sh @@ -53,6 +54,6 @@ install-tests += $(foreach x, $(nix_tests), tests/$(x)) tests-environment = NIX_REMOTE= $(bash) -e -clean-files += $(d)/common.sh $(d)/config.nix +clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix -test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT) +test-deps += tests/common.sh tests/config.nix tests/ca/config.nix tests/plugins/libplugintest.$(SO_EXT) From ca96f5219489c1002499bfe2c580fdd458219144 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 17 May 2021 17:49:20 +0200 Subject: [PATCH 033/555] Split the parsing of an `App` and its resolving That way things (like `nix flake check`) can evaluate the `app` outputs without having to build anything --- src/libcmd/installables.hh | 8 ++++- src/nix/app.cc | 66 +++++++++++++++++++------------------- src/nix/bundle.cc | 3 +- src/nix/run.cc | 4 +-- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 16bc4ae68..298fd48f8 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -23,6 +23,12 @@ struct App // FIXME: add args, sandbox settings, metadata, ... }; +struct UnresolvedApp +{ + App unresolved; + App resolve(ref); +}; + struct Installable { virtual ~Installable() { } @@ -33,7 +39,7 @@ struct Installable DerivedPath toDerivedPath(); - App toApp(EvalState & state); + UnresolvedApp toApp(EvalState & state); virtual std::pair toValue(EvalState & state) { diff --git a/src/nix/app.cc b/src/nix/app.cc index 1e30deb34..bdc64a886 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -58,33 +58,24 @@ std::string resolveString(Store & store, const std::string & toResolve, const Bu return rewriteStrings(toResolve, rewrites); } -App Installable::toApp(EvalState & state) +UnresolvedApp Installable::toApp(EvalState & state) { auto [cursor, attrPath] = getCursor(state); auto type = cursor->getAttr("type")->getString(); - auto checkProgram = [&](const Path & program) - { - if (!state.store->isInStore(program)) - throw Error("app program '%s' is not in the Nix store", program); - }; - - std::vector> context; - std::string unresolvedProgram; - - if (type == "app") { - auto [program, context_] = cursor->getAttr("program")->getStringWithContext(); - unresolvedProgram = program; + auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - for (auto & [path, name] : context_) - context.push_back(std::make_shared( - state.store, - DerivedPathBuilt{ - .drvPath = state.store->parseStorePath(path), - .outputs = {name}, - })); + + std::vector context2; + for (auto & [path, name] : context) + context2.push_back({state.store->parseStorePath(path), {name}}); + + return UnresolvedApp{App { + .context = std::move(context2), + .program = program, + }}; } else if (type == "derivation") { @@ -98,24 +89,33 @@ App Installable::toApp(EvalState & state) aMainProgram ? aMainProgram->getString() : DrvName(name).name; - unresolvedProgram = outPath + "/bin/" + mainProgram; - context = {std::make_shared( - state.store, - DerivedPathBuilt{ - .drvPath = drvPath, - .outputs = {outputName}, - })}; + auto program = outPath + "/bin/" + mainProgram; + return UnresolvedApp { App { + .context = { { drvPath, {outputName} } }, + .program = program, + }}; } else throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); +} - auto builtContext = build(state.store, Realise::Outputs, context); - auto program = resolveString(*state.store, unresolvedProgram, builtContext); - checkProgram(program); - return App { - .program = program, - }; +App UnresolvedApp::resolve(ref store) +{ + auto res = unresolved; + + std::vector> installableContext; + + for (auto & ctxElt : unresolved.context) + installableContext.push_back( + std::make_shared(store, ctxElt.toDerivedPath())); + + auto builtContext = build(store, Realise::Outputs, installableContext); + res.program = resolveString(*store, unresolved.program, builtContext); + if (store->isInStore(res.program)) + throw Error("app program '%s' is not in the Nix store", res.program); + + return res; } } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 53dccc63a..88bc3d1d1 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -69,8 +69,7 @@ struct CmdBundle : InstallableCommand { auto evalState = getEvalState(); - auto app = installable->toApp(*evalState); - store->buildPaths(toDerivedPaths(app.context)); + auto app = installable->toApp(*evalState).resolve(store); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; diff --git a/src/nix/run.cc b/src/nix/run.cc index b5d8ab38a..f684c5ea4 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -178,9 +178,7 @@ struct CmdRun : InstallableCommand, RunCommon { auto state = getEvalState(); - auto app = installable->toApp(*state); - - state->store->buildPaths(toDerivedPaths(app.context)); + auto app = installable->toApp(*state).resolve(store); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 5fd8cf76676a280ae2b7a86ddabc6b14b41ebfe5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 17 May 2021 15:07:00 -0500 Subject: [PATCH 034/555] Source bashrc first in nix develop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ~/.bashrc should be sourced first in the rc script so that PATH & other env vars give precedence over the bashrc PATH. Also, in my bashrc I alias rm as: alias rm='rm -Iv' To avoid running this alias (which shows ‘removed '/tmp/nix-shell.*'), we can just prefix rm with command. --- src/nix/develop.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 2b64d9a31..10f843651 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -395,7 +395,7 @@ struct CmdDevelop : Common, MixEnvironment if (verbosity >= lvlDebug) script += "set -x\n"; - script += fmt("rm -f '%s'\n", rcFilePath); + script += fmt("command rm -f '%s'\n", rcFilePath); if (phase) { if (!command.empty()) @@ -414,7 +414,7 @@ struct CmdDevelop : Common, MixEnvironment } else { - script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); if (developSettings.bashPromptSuffix != "") From 59d0de6ea17b7836e7f2e169ea4c8e65b413500a Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 18 May 2021 13:54:05 +0200 Subject: [PATCH 035/555] Restore an accidentally suppressed negation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accidentally removed in ca96f5219489c1002499bfe2c580fdd458219144. This caused `nix run` to systematically fail with ``` error: app program '/nix/store/…' is not in the Nix store ``` --- src/nix/app.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/app.cc b/src/nix/app.cc index bdc64a886..01a0064db 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -112,7 +112,7 @@ App UnresolvedApp::resolve(ref store) auto builtContext = build(store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); - if (store->isInStore(res.program)) + if (!store->isInStore(res.program)) throw Error("app program '%s' is not in the Nix store", res.program); return res; From 3d90ab9345ea1379fa3aea2a26f8eaa498f3e3d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 18 May 2021 16:38:55 -0500 Subject: [PATCH 036/555] Fix extra slash in canonPath output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When you have a symlink like: /tmp -> ./private/tmp you need to resolve ./private/tmp relative to /tmp’s dir: ‘/’. Unlike any other path output by dirOf, / ends with a slash. We don’t want trailing slashes here since we will append another slash in the next comoponent, so clear s like we would if it was a symlink to an absoute path. This should fix at least part of the issue in https://github.com/NixOS/nix/issues/4822, will need confirmation that it actually fixes the problem to close though. Introduced in f3f228700a52857fe6e8632df4e935551ea219ff. --- src/libutil/util.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5f597bf06..7e57fd7ca 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -155,6 +155,9 @@ Path canonPath(const Path & path, bool resolveSymlinks) s.clear(); /* restart for symlinks pointing to absolute path */ } else { s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } } } } From 184558834a475955e6c5d4b38745544c2c2907a3 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 18 May 2021 14:30:32 +0200 Subject: [PATCH 037/555] Extract a generic `computeClosure` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the `closure` logic of `computeFSClosure` to its own (templated) function. This doesn’t bring much by itself (except for the ability to properly test the “closure” functionality independently from the rest), but it allows reusing it (in particular for the realisations which will require a very similar closure computation) --- src/libstore/misc.cc | 133 ++++++++++++++--------------------- src/libutil/closure.hh | 69 ++++++++++++++++++ src/libutil/tests/closure.cc | 70 ++++++++++++++++++ 3 files changed, 193 insertions(+), 79 deletions(-) create mode 100644 src/libutil/closure.hh create mode 100644 src/libutil/tests/closure.cc diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a99a2fc78..bc5fd968c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -6,98 +6,73 @@ #include "thread-pool.hh" #include "topo-sort.hh" #include "callback.hh" +#include "closure.hh" namespace nix { - void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - struct State - { - size_t pending; - StorePathSet & paths; - std::exception_ptr exc; - }; + std::function(const StorePath & path, std::future> &)> queryDeps; + if (flipDirection) + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + StorePathSet referrers; + queryReferrers(path, referrers); + for (auto& ref : referrers) + if (ref != path) + res.insert(ref); - Sync state_(State{0, paths_, 0}); + if (includeOutputs) + for (auto& i : queryValidDerivers(path)) + res.insert(i); - std::function enqueue; + if (includeDerivers && path.isDerivation()) + for (auto& i : queryDerivationOutputs(path)) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + res.insert(i); + return res; + }; + else + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + auto info = fut.get(); + for (auto& ref : info->references) + if (ref != path) + res.insert(ref); - std::condition_variable done; + if (includeOutputs && path.isDerivation()) + for (auto& i : queryDerivationOutputs(path)) + if (isValidPath(i)) + res.insert(i); - enqueue = [&](const StorePath & path) -> void { - { - auto state(state_.lock()); - if (state->exc) return; - if (!state->paths.insert(path).second) return; - state->pending++; - } + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + res.insert(*info->deriver); + return res; + }; - queryPathInfo(path, {[&](std::future> fut) { - // FIXME: calls to isValidPath() should be async - - try { - auto info = fut.get(); - - if (flipDirection) { - - StorePathSet referrers; - queryReferrers(path, referrers); - for (auto & ref : referrers) - if (ref != path) - enqueue(ref); - - if (includeOutputs) - for (auto & i : queryValidDerivers(path)) - enqueue(i); - - if (includeDerivers && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); - - } else { - - for (auto & ref : info->references) - if (ref != path) - enqueue(ref); - - if (includeOutputs && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); - - if (includeDerivers && info->deriver && isValidPath(*info->deriver)) - enqueue(*info->deriver); - - } - - { - auto state(state_.lock()); - assert(state->pending); - if (!--state->pending) done.notify_one(); - } - - } catch (...) { - auto state(state_.lock()); - if (!state->exc) state->exc = std::current_exception(); - assert(state->pending); - if (!--state->pending) done.notify_one(); - }; - }}); - }; - - for (auto & startPath : startPaths) - enqueue(startPath); - - { - auto state(state_.lock()); - while (state->pending) state.wait(done); - if (state->exc) std::rethrow_exception(state->exc); - } + computeClosure( + startPaths, paths_, + [&](const StorePath& path, + std::function>&)> + processEdges) { + std::promise> promise; + std::function>)> + getDependencies = + [&](std::future> fut) { + try { + promise.set_value(queryDeps(path, fut)); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }; + queryPathInfo(path, getDependencies); + processEdges(promise); + }); } - void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh new file mode 100644 index 000000000..779b9b2d5 --- /dev/null +++ b/src/libutil/closure.hh @@ -0,0 +1,69 @@ +#include +#include +#include "sync.hh" + +using std::set; + +namespace nix { + +template +using GetEdgesAsync = std::function> &)>)>; + +template +void computeClosure( + const set startElts, + set & res, + GetEdgesAsync getEdgesAsync +) +{ + struct State + { + size_t pending; + set & res; + std::exception_ptr exc; + }; + + Sync state_(State{0, res, 0}); + + std::function enqueue; + + std::condition_variable done; + + enqueue = [&](const T & current) -> void { + { + auto state(state_.lock()); + if (state->exc) return; + if (!state->res.insert(current).second) return; + state->pending++; + } + + getEdgesAsync(current, [&](std::promise> & prom) { + try { + auto children = prom.get_future().get(); + for (auto & child : children) + enqueue(child); + { + auto state(state_.lock()); + assert(state->pending); + if (!--state->pending) done.notify_one(); + } + } catch (...) { + auto state(state_.lock()); + if (!state->exc) state->exc = std::current_exception(); + assert(state->pending); + if (!--state->pending) done.notify_one(); + }; + }); + }; + + for (auto & startElt : startElts) + enqueue(startElt); + + { + auto state(state_.lock()); + while (state->pending) state.wait(done); + if (state->exc) std::rethrow_exception(state->exc); + } +} + +} diff --git a/src/libutil/tests/closure.cc b/src/libutil/tests/closure.cc new file mode 100644 index 000000000..7597e7807 --- /dev/null +++ b/src/libutil/tests/closure.cc @@ -0,0 +1,70 @@ +#include "closure.hh" +#include + +namespace nix { + +using namespace std; + +map> testGraph = { + { "A", { "B", "C", "G" } }, + { "B", { "A" } }, // Loops back to A + { "C", { "F" } }, // Indirect reference + { "D", { "A" } }, // Not reachable, but has backreferences + { "E", {} }, // Just not reachable + { "F", {} }, + { "G", { "G" } }, // Self reference +}; + +TEST(closure, correctClosure) { + set aClosure; + set expectedClosure = {"A", "B", "C", "F", "G"}; + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promisedNodes; + promisedNodes.set_value(testGraph[currentNode]); + processEdges(promisedNodes); + } + ); + + ASSERT_EQ(aClosure, expectedClosure); +} + +TEST(closure, properlyHandlesDirectExceptions) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + throw TestExn(); + } + ), + TestExn + ); +} + +TEST(closure, properlyHandlesExceptionsInPromise) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promise; + try { + throw TestExn(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + processEdges(promise); + } + ), + TestExn + ); +} + +} From a8416866cf76128f212f2a9caba0d439b0652115 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 7 May 2021 11:06:17 +0200 Subject: [PATCH 038/555] Always send the realisations as JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align all the worker protocol with `buildDerivation` which inlines the realisations as one opaque json blob. That way we don’t have to bother changing the remote store protocol when the definition of `Realisation` changes, as long as we keep the json backwards-compatible --- src/libstore/daemon.cc | 25 ++++++++++++++++++------- src/libstore/remote-store.cc | 23 +++++++++++++++++------ src/libstore/worker-protocol.hh | 2 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 72b3e3e13..e06fb9ce2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref store, case wopRegisterDrvOutput: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); - store->registerDrvOutput(Realisation{ - .id = outputId, .outPath = outputPath}); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + } else { + auto realisation = worker_proto::read(*store, from, Phantom()); + store->registerDrvOutput(realisation); + } logger->stopWork(); break; } @@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref store, auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - std::set outPaths; - if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + } else { + std::set realisations; + if (info) realisations.insert(*info); + worker_proto::write(*store, to, realisations); + } break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d9b6e9488..aec243637 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -653,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << wopRegisterDrvOutput; - conn->to << info.id.to_string(); - conn->to << std::string(info.outPath.to_string()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + } else { + worker_proto::write(*this, conn->to, info); + } conn.processStderr(); } @@ -664,10 +668,17 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + } else { + auto realisations = worker_proto::read(*this, conn->from, Phantom>{}); + if (realisations.empty()) + return std::nullopt; + return *realisations.begin(); + } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index fdd692cf0..e89183d40 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 30) +#define PROTOCOL_VERSION (1 << 8 | 31) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From 9e9a8456d7d111e76ad6010d58b11389611ffb17 Mon Sep 17 00:00:00 2001 From: Michael Adler Date: Mon, 24 May 2021 12:35:39 +0200 Subject: [PATCH 039/555] fix doc: nix profile info -> nix profile list Signed-off-by: Michael Adler --- src/nix/profile-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 2bd5d256d..e06e74abe 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -18,7 +18,7 @@ R""( * Upgrade a specific profile element by number: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … # nix profile upgrade 0 From 79ae9e4558cbefd743f28a5e73110c2303b03a85 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 25 May 2021 10:29:10 +0200 Subject: [PATCH 040/555] Make the Nar hash non modulo It makes much more sense to have the Nar hash be a plain straight hash rather than a hash modulo --- src/libstore/build/local-derivation-goal.cc | 38 +++++++++++---------- src/libstore/local-store.cc | 10 ++---- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3de12c6ff..28981a1a1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2354,32 +2354,19 @@ void LocalDerivationGoal::registerOutputs() } auto got = caSink.finish().first; auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( + + auto finalPath = worker.store.makeFixedOutputPath( outputHash.method, got, outputPathName(drv->name, outputName), refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { + refs.first); + if (scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); rsink2(*sink.s); rsink2.flush(); }); @@ -2389,6 +2376,21 @@ void LocalDerivationGoal::registerOutputs() movePath(tmpPath, actualPath); } + HashResult narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { + finalPath, + narHashAndSize.first, + }; + + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + assert(newInfo0.ca); return newInfo0; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 83daa7506..f29df8bad 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1152,17 +1152,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) - hashSink = std::make_unique(htSHA256); - else - hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); + HashSink hashSink(htSHA256); - TeeSource wrapperSource { source, *hashSink }; + TeeSource wrapperSource { source, hashSink }; restorePath(realPath, wrapperSource); - auto hashResult = hashSink->finish(); + auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", From 129384bcf3bf903ef1c6661b3f21659a2ad94228 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 26 May 2021 09:39:29 +0200 Subject: [PATCH 041/555] Remove the remaining occurenceses of a NarHash modulo --- src/libstore/local-store.cc | 10 +++------- src/nix/verify.cc | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f29df8bad..3b99ba39d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1436,14 +1436,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - dumpPath(Store::toRealPath(i), *hashSink); - auto current = hashSink->finish(); + dumpPath(Store::toRealPath(i), hashSink); + auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 1721c7f16..f5a576064 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -97,15 +97,11 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - std::unique_ptr hashSink; - if (!info->ca) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - store->narFromPath(info->path, *hashSink); + store->narFromPath(info->path, hashSink); - auto hash = hashSink->finish(); + auto hash = hashSink.finish(); if (hash.first != info->narHash) { corrupted++; From 7616268812cc2fe141132749d6d667b5c4f4d877 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 7 May 2021 11:06:17 +0200 Subject: [PATCH 042/555] Always send the realisations as JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align all the worker protocol with `buildDerivation` which inlines the realisations as one opaque json blob. That way we don’t have to bother changing the remote store protocol when the definition of `Realisation` changes, as long as we keep the json backwards-compatible --- src/libstore/daemon.cc | 25 ++++++++++++++++++------- src/libstore/remote-store.cc | 23 +++++++++++++++++------ src/libstore/worker-protocol.hh | 2 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 72b3e3e13..e06fb9ce2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref store, case wopRegisterDrvOutput: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); - store->registerDrvOutput(Realisation{ - .id = outputId, .outPath = outputPath}); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + } else { + auto realisation = worker_proto::read(*store, from, Phantom()); + store->registerDrvOutput(realisation); + } logger->stopWork(); break; } @@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref store, auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - std::set outPaths; - if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + } else { + std::set realisations; + if (info) realisations.insert(*info); + worker_proto::write(*store, to, realisations); + } break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d9b6e9488..aec243637 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -653,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << wopRegisterDrvOutput; - conn->to << info.id.to_string(); - conn->to << std::string(info.outPath.to_string()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + } else { + worker_proto::write(*this, conn->to, info); + } conn.processStderr(); } @@ -664,10 +668,17 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + } else { + auto realisations = worker_proto::read(*this, conn->from, Phantom>{}); + if (realisations.empty()) + return std::nullopt; + return *realisations.begin(); + } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index fdd692cf0..e89183d40 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 30) +#define PROTOCOL_VERSION (1 << 8 | 31) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From 7ce0441d806d12bb073047337545e3901404996d Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 10 Nov 2020 14:49:25 +0100 Subject: [PATCH 043/555] Add a dependencies field to DrvOutputInfo Currently never used, nor set but will be useful shortly --- src/libstore/realisation.cc | 10 ++++++++++ src/libstore/realisation.hh | 2 ++ 2 files changed, 12 insertions(+) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 638065547..27ad6c150 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -22,10 +22,14 @@ std::string DrvOutput::to_string() const { } nlohmann::json Realisation::toJSON() const { + nlohmann::json jsonDrvOutputDeps; + for (auto & dep : drvOutputDeps) + jsonDrvOutputDeps.push_back(dep.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, {"signatures", signatures}, + {"drvOutputDeps", jsonDrvOutputDeps}, }; } @@ -51,10 +55,16 @@ Realisation Realisation::fromJSON( if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + std::set drvOutputDeps; + if (auto jsonDependencies = json.find("drvOutputDeps"); jsonDependencies != json.end()) + for (auto & jsonDep : *jsonDependencies) + drvOutputDeps.insert(DrvOutput::parse(jsonDep.get())); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), .signatures = signatures, + .drvOutputDeps = drvOutputDeps, }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f5049c9e9..1e2808ce3 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -28,6 +28,8 @@ struct Realisation { StringSet signatures; + std::set drvOutputDeps; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); From eca6ff06d611ef005e80d419e1b6050393fc056d Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 7 May 2021 13:57:01 +0200 Subject: [PATCH 044/555] Store the realisation deps on the local store --- src/libstore/ca-specific-schema.sql | 11 +++++- src/libstore/local-store.cc | 61 +++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 20ee046a1..08af0cc1f 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -3,10 +3,19 @@ -- is enabled create table if not exists Realisations ( + id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, signatures text, -- space-separated list - primary key (drvPath, outputName), foreign key (outputPath) references ValidPaths(id) on delete cascade ); + +create index if not exists IndexRealisations on Realisations(drvPath, outputName); + +create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict +); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 83daa7506..f8d55621d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -59,6 +59,8 @@ struct LocalStore::State::Stmts { SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; + SQLiteStmt QueryRealisationRealisationReferences; + SQLiteStmt AddRealisationRealisationReference; }; int getSchema(Path schemaPath) @@ -316,7 +318,7 @@ LocalStore::LocalStore(const Params & params) )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path, Realisations.signatures from Realisations + select Realisations.id, Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -328,6 +330,19 @@ LocalStore::LocalStore(const Params & params) where drvPath = ? ; )"); + state->stmts->QueryRealisationRealisationReferences.create(state->db, + R"( + select drvPath, outputName from Realisations + join RealisationsRefs on realisationReference = Realisations.id + where referrer = ?; + )"); + state->stmts->AddRealisationRealisationReference.create(state->db, + R"( + insert or replace into RealisationsRefs (referrer, realisationReference) + values ( + ?, + (select id from Realisations where drvPath = ? and outputName = ?)); + )"); } } @@ -666,13 +681,17 @@ void LocalStore::registerDrvOutput(const Realisation & info) settings.requireExperimentalFeature("ca-derivations"); auto state(_state.lock()); retrySQLite([&]() { - state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) + state->stmts->RegisterRealisedOutput + .use()(info.id.strHash())(info.id.outputName)(printStorePath( + info.outPath))(concatStringsSep(" ", info.signatures)) .exec(); }); + uint64_t myId = state->db.getLastInsertedRowId(); + for (auto& outputId : info.drvOutputDeps) { + state->stmts->AddRealisationRealisationReference + .use()(myId)(outputId.strHash())(outputId.outputName) + .exec(); + } } void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) @@ -1670,14 +1689,32 @@ std::optional LocalStore::queryRealisation( typedef std::optional Ret; return retrySQLite([&]() -> Ret { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( - id.outputName)); - if (!use.next()) + auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()( + id.strHash())(id.outputName)); + if (!useQueryRealisedOutput.next()) return std::nullopt; - auto outputPath = parseStorePath(use.getStr(0)); - auto signatures = tokenizeString(use.getStr(1)); + auto realisationDbId = useQueryRealisedOutput.getInt(0); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto signatures = + tokenizeString(useQueryRealisedOutput.getStr(2)); + + std::set drvOutputDeps; + auto useRealisationRefs( + state->stmts->QueryRealisationRealisationReferences.use()( + realisationDbId)); + while (useRealisationRefs.next()) + drvOutputDeps.insert(DrvOutput{ + Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), + useRealisationRefs.getStr(1), + } + ); + return Ret{Realisation{ - .id = id, .outPath = outputPath, .signatures = signatures}}; + .id = id, + .outPath = outputPath, + .signatures = signatures, + .drvOutputDeps = drvOutputDeps, + }}; }); } } // namespace nix From af3afd25eafb7866406aee04ef049121a3e3ffb0 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 10:26:58 +0200 Subject: [PATCH 045/555] Add a method to compute the closure of a realisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only considers the closure in term of `Realisation`, ignores all the opaque inputs. Dunno whether that’s the nicest solution, need to think it through a bit --- src/libstore/realisation.cc | 38 +++++++++++++++++++++++++++++++++++++ src/libstore/realisation.hh | 3 +++ 2 files changed, 41 insertions(+) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 27ad6c150..fab10f68c 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include "closure.hh" #include namespace nix { @@ -21,6 +22,43 @@ std::string DrvOutput::to_string() const { return strHash() + "!" + outputName; } +std::set Realisation::closure(Store & store, std::set startOutputs) +{ + std::set res; + Realisation::closure(store, startOutputs, res); + return res; +} + +void Realisation::closure(Store & store, std::set startOutputs, std::set & res) +{ + auto getDeps = [&](const Realisation& current) -> std::set { + std::set res; + for (auto& currentDep : current.drvOutputDeps) { + if (auto currentRealisation = store.queryRealisation(currentDep)) + res.insert(*currentRealisation); + else + throw Error( + "Unrealised derivation '%s'", currentDep.to_string()); + } + return res; + }; + + computeClosure( + startOutputs, res, + [&](const Realisation& current, + std::function>&)> + processEdges) { + std::promise> promise; + try { + auto res = getDeps(current); + promise.set_value(res); + } catch (...) { + promise.set_exception(std::current_exception()); + } + return processEdges(promise); + }); +} + nlohmann::json Realisation::toJSON() const { nlohmann::json jsonDrvOutputDeps; for (auto & dep : drvOutputDeps) diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 1e2808ce3..776ab606c 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -38,6 +38,9 @@ struct Realisation { bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; + static std::set closure(Store &, std::set); + static void closure(Store &, std::set, std::set& res); + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); From 8c30acc3e896839283a2c96ec97cc0c811e8ad6a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 10:35:31 +0200 Subject: [PATCH 046/555] Properly track the drvoutput references when building --- src/libstore/build/derivation-goal.cc | 1 + src/libstore/misc.cc | 42 +++++++++++++++++++++++++++ src/libstore/store-api.hh | 5 ++++ 3 files changed, 48 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 9100d3333..93fc54440 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -927,6 +927,7 @@ void DerivationGoal::resolvedFinished() { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.signatures.clear(); + newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bc5fd968c..8793f9f4a 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -254,5 +254,47 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } +std::set drvOutputReferences( + const std::set inputRealisations, + const StorePathSet pathReferences) +{ + std::set res; + std::map> inputsByOutputPath; + for (const auto & input : inputRealisations) + inputsByOutputPath[input.outPath].insert(input.id); + + for (const auto & path : pathReferences) { + auto theseInputs = inputsByOutputPath[path]; + res.insert(theseInputs.begin(), theseInputs.end()); + } + + return res; +} + +std::set drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath) +{ + std::set inputRealisations; + + for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto& outputName : outputNames) { + auto thisRealisation = store.queryRealisation( + DrvOutput{outputHashes.at(outputName), outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } + } + + auto info = store.queryPathInfo(outputPath); + + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); +} } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f66298991..29af0f495 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -864,4 +864,9 @@ std::pair splitUriAndParams(const std::string & uri) std::optional getDerivationCA(const BasicDerivation & drv); +std::set drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath); + } From 63ebfc73c56978665bf6c58c3e4ad84485355b1a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 10:36:48 +0200 Subject: [PATCH 047/555] Make `copyPaths` copy the whole realisations closure Otherwise registering the realisations on the remote side might fail as it now expects a complete closure --- src/libstore/store-api.cc | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 93fcb068f..08573f709 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -780,20 +780,40 @@ std::map copyPaths(ref srcStore, ref dstStor RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { StorePathSet storePaths; - std::set realisations; + std::set toplevelRealisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { settings.requireExperimentalFeature("ca-derivations"); - realisations.insert(*realisation); + toplevelRealisations.insert(*realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + + ThreadPool pool; + try { - for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation, checkSigs); - } - } catch (MissingExperimentalFeature & e) { + // Copy the realisation closure + processGraph( + pool, Realisation::closure(*srcStore, toplevelRealisations), + [&](const Realisation& current) -> std::set { + std::set children; + for (const auto& drvOutput : current.drvOutputDeps) { + auto currentChild = srcStore->queryRealisation(drvOutput); + if (!currentChild) + throw Error( + "Incomplete realisation closure: '%s' is a " + "dependency " + "of '%s' but isn’t registered", + drvOutput.to_string(), current.id.to_string()); + children.insert(*currentChild); + } + return children; + }, + [&](const Realisation& current) -> void { + dstStore->registerDrvOutput(current, checkSigs); + }); + } catch (MissingExperimentalFeature& e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. From cb46d70794b8677b7927e6ae3d492e6db886c7aa Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 12:30:12 +0200 Subject: [PATCH 048/555] Add a db migration script --- src/libstore/local-store.cc | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f8d55621d..ce0e94865 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -78,7 +78,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 1; + const int nixCASchemaVersion = 2; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -96,7 +96,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) #include "ca-specific-schema.sql.gen.hh" ; db.exec(schema); + curCASchema = nixCASchemaVersion; } + + if (curCASchema < 2) { + SQLiteTxn txn(db); + // Ugly little sql dance to add a new `id` column and make it the primary key + db.exec(R"( + create table Realisations2 ( + id integer primary key autoincrement not null, + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + signatures text, -- space-separated list + foreign key (outputPath) references ValidPaths(id) on delete cascade + ); + insert into Realisations2 (drvPath, outputName, outputPath, signatures) + select drvPath, outputName, outputPath, signatures from Realisations; + drop table Realisations; + alter table Realisations2 rename to Realisations; + )"); + db.exec(R"( + create index if not exists IndexRealisations on Realisations(drvPath, outputName); + + create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict + ); + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } From 1f3ff0d193c270f7b97af4aa3e463be01dbe5f2d Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 26 May 2021 16:09:02 +0200 Subject: [PATCH 049/555] Aso track the output path of the realisation dependencies --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/local-store.cc | 34 +++++++++++++++------------ src/libstore/misc.cc | 17 ++++++-------- src/libstore/realisation.cc | 20 ++++++++-------- src/libstore/realisation.hh | 8 ++++++- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 +- 7 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93fc54440..8c9ef0101 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -927,7 +927,7 @@ void DerivationGoal::resolvedFinished() { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.signatures.clear(); - newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ce0e94865..debe70037 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -711,19 +711,19 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { settings.requireExperimentalFeature("ca-derivations"); - auto state(_state.lock()); retrySQLite([&]() { + auto state(_state.lock()); state->stmts->RegisterRealisedOutput .use()(info.id.strHash())(info.id.outputName)(printStorePath( info.outPath))(concatStringsSep(" ", info.signatures)) .exec(); + uint64_t myId = state->db.getLastInsertedRowId(); + for (auto & [outputId, _] : info.dependentRealisations) { + state->stmts->AddRealisationRealisationReference + .use()(myId)(outputId.strHash())(outputId.outputName) + .exec(); + } }); - uint64_t myId = state->db.getLastInsertedRowId(); - for (auto& outputId : info.drvOutputDeps) { - state->stmts->AddRealisationRealisationReference - .use()(myId)(outputId.strHash())(outputId.outputName) - .exec(); - } } void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) @@ -1730,22 +1730,26 @@ std::optional LocalStore::queryRealisation( auto signatures = tokenizeString(useQueryRealisedOutput.getStr(2)); - std::set drvOutputDeps; + std::map dependentRealisations; auto useRealisationRefs( state->stmts->QueryRealisationRealisationReferences.use()( realisationDbId)); - while (useRealisationRefs.next()) - drvOutputDeps.insert(DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - } - ); + while (useRealisationRefs.next()) { + auto depHash = useRealisationRefs.getStr(0); + auto depOutputName = useRealisationRefs.getStr(1); + auto useQueryRealisedOutput( + state->stmts->QueryRealisedOutput.use()(depHash)(depOutputName)); + assert(useQueryRealisedOutput.next()); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName }; + dependentRealisations.insert({depId, outputPath}); + } return Ret{Realisation{ .id = id, .outPath = outputPath, .signatures = signatures, - .drvOutputDeps = drvOutputDeps, + .dependentRealisations = dependentRealisations, }}; }); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 8793f9f4a..80ee15c49 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -254,25 +254,22 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } -std::set drvOutputReferences( +std::map drvOutputReferences( const std::set inputRealisations, const StorePathSet pathReferences) { - std::set res; + std::map res; - std::map> inputsByOutputPath; - for (const auto & input : inputRealisations) - inputsByOutputPath[input.outPath].insert(input.id); - - for (const auto & path : pathReferences) { - auto theseInputs = inputsByOutputPath[path]; - res.insert(theseInputs.begin(), theseInputs.end()); + for (const auto & input : inputRealisations) { + if (pathReferences.count(input.outPath)) { + res.insert({input.id, input.outPath}); + } } return res; } -std::set drvOutputReferences( +std::map drvOutputReferences( Store & store, const Derivation & drv, const StorePath & outputPath) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index fab10f68c..d2d306476 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -33,7 +33,7 @@ void Realisation::closure(Store & store, std::set startOutputs, std { auto getDeps = [&](const Realisation& current) -> std::set { std::set res; - for (auto& currentDep : current.drvOutputDeps) { + for (auto& [currentDep, _] : current.dependentRealisations) { if (auto currentRealisation = store.queryRealisation(currentDep)) res.insert(*currentRealisation); else @@ -60,14 +60,14 @@ void Realisation::closure(Store & store, std::set startOutputs, std } nlohmann::json Realisation::toJSON() const { - nlohmann::json jsonDrvOutputDeps; - for (auto & dep : drvOutputDeps) - jsonDrvOutputDeps.push_back(dep.to_string()); + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, {"signatures", signatures}, - {"drvOutputDeps", jsonDrvOutputDeps}, + {"dependentRealisations", jsonDependentRealisations}, }; } @@ -93,16 +93,16 @@ Realisation Realisation::fromJSON( if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); - std::set drvOutputDeps; - if (auto jsonDependencies = json.find("drvOutputDeps"); jsonDependencies != json.end()) - for (auto & jsonDep : *jsonDependencies) - drvOutputDeps.insert(DrvOutput::parse(jsonDep.get())); + std::map dependentRealisations; + if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) + for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get>()) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), .signatures = signatures, - .drvOutputDeps = drvOutputDeps, + .dependentRealisations = dependentRealisations, }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 776ab606c..8cda5a752 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -28,7 +28,13 @@ struct Realisation { StringSet signatures; - std::set drvOutputDeps; + /** + * The realisations that are required for the current one to be valid. + * + * When importing this realisation, the store will first check that all its + * dependencies exist, and map to the correct output path + */ + std::map dependentRealisations; nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 08573f709..6f29c430a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -798,7 +798,7 @@ std::map copyPaths(ref srcStore, ref dstStor pool, Realisation::closure(*srcStore, toplevelRealisations), [&](const Realisation& current) -> std::set { std::set children; - for (const auto& drvOutput : current.drvOutputDeps) { + for (const auto& [drvOutput, _] : current.dependentRealisations) { auto currentChild = srcStore->queryRealisation(drvOutput); if (!currentChild) throw Error( diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 29af0f495..9a0027641 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -864,7 +864,7 @@ std::pair splitUriAndParams(const std::string & uri) std::optional getDerivationCA(const BasicDerivation & drv); -std::set drvOutputReferences( +std::map drvOutputReferences( Store & store, const Derivation & drv, const StorePath & outputPath); From a22755721b51949dbf03bc1eb156d6f37dc10bac Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 14:51:34 +0200 Subject: [PATCH 050/555] Recursively substitute the realisations Make sure that whenever we substitute a realisation, we also substitute its entire closure --- src/libstore/build/drv-output-substitution-goal.cc | 6 ++++++ src/libutil/comparator.hh | 4 +++- tests/ca/substitute.sh | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index a5ac4c49d..1703e845d 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -53,6 +53,12 @@ void DrvOutputSubstitutionGoal::tryNext() return; } + for (const auto & [drvOutputDep, _] : outputInfo->dependentRealisations) { + if (drvOutputDep != id) { + addWaitee(worker.makeDrvOutputSubstitutionGoal(drvOutputDep)); + } + } + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); if (waitees.empty()) outPathValid(); diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 0315dc506..eecd5b819 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -25,6 +25,8 @@ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) + GENERATE_LEQ(args) \ + GENERATE_NEQ(args) diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index dfc4ea68e..c80feaacf 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -17,11 +17,15 @@ buildDrvs () { # Populate the remote cache clearStore -buildDrvs --post-build-hook ../push-to-store.sh +nix copy --to $REMOTE_STORE --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA +# Check that the thing we’ve just substituted has its realisation stored +nix realisation info --file ./content-addressed.nix transitivelyDependentCA +# Check that its dependencies have it too +nix realisation info --file ./content-addressed.nix dependentCA rootCA # Same thing, but # 1. With non-ca derivations From 822e338e5ccee59963f50c876998c2e55ee8caa7 Mon Sep 17 00:00:00 2001 From: Patrick Hilhorst Date: Thu, 27 May 2021 21:48:39 +0200 Subject: [PATCH 051/555] throw freenode down the memory hole --- .github/STALE-BOT.md | 2 +- README.md | 3 ++- scripts/install-multi-user.sh | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 5e8f5d929..383717bfc 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -3,7 +3,7 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. - _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos). +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org). ## Suggestions for PRs diff --git a/README.md b/README.md index 4686010ef..80d6f128c 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ build nix from source with nix-build or how to get a development environment. - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) -- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) +- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9b62e708f..e1046c19c 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -63,7 +63,8 @@ contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" echo "Or feel free to contact the team:" - echo " - IRC: in #nixos on irc.freenode.net" + echo " - Matrix: #nix:nixos.org" + echo " - IRC: in #nixos on irc.libera.chat" echo " - twitter: @nixos_org" echo " - forum: https://discourse.nixos.org" } From 4da9ec772c54f17ac04bbc5bcb75140d13a29500 Mon Sep 17 00:00:00 2001 From: Timothy Klim Date: Sat, 29 May 2021 16:03:26 +0700 Subject: [PATCH 052/555] Add .tar.zst support for TarballInputScheme --- src/libfetchers/tarball.cc | 3 ++- src/nix/flake.md | 4 ++-- tests/lang/parse-okay-url.nix | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b8d7d2c70..257465bae 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2")) + && !hasSuffix(url.path, ".tar.bz2") + && !hasSuffix(url.path, ".tar.zst")) return {}; Input input; diff --git a/src/nix/flake.md b/src/nix/flake.md index 9e936a049..3d273100b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -186,8 +186,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` - or `.tar.bz2`. + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, + `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: diff --git a/tests/lang/parse-okay-url.nix b/tests/lang/parse-okay-url.nix index fce3b13ee..08de27d0a 100644 --- a/tests/lang/parse-okay-url.nix +++ b/tests/lang/parse-okay-url.nix @@ -3,5 +3,6 @@ http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2 http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz + https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz ] From 1f390922d0e34382265205f6895c2b71c5a8e6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sat, 29 May 2021 19:40:56 +0200 Subject: [PATCH 053/555] Build for aarch64-darwin --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ebaafb049..a445c7763 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,7 @@ linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" ]; + systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); From f674f7f43417c5440aecf54fbaa922b0fdce1ec4 Mon Sep 17 00:00:00 2001 From: Jeremy Schlatter Date: Tue, 1 Jun 2021 00:05:34 -0700 Subject: [PATCH 054/555] Fix typo in documentation --- src/nix/flake-check.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index dc079ba0c..ffe9d64b4 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -24,7 +24,7 @@ built successfully. # Evaluation checks -This following flake output attributes must be derivations: +The following flake output attributes must be derivations: * `checks.`*system*`.`*name* * `defaultPackage.`*system*` From 1fefe808f65f9545f2f112229bf38adc626aed07 Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Thu, 27 May 2021 12:05:13 +0200 Subject: [PATCH 055/555] enable aarch64-darwin build disable lowdown sandbox on aarch64-darwin --- flake.lock | 8 ++++---- flake.nix | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 06c507e7d..325e01fb8 100644 --- a/flake.lock +++ b/flake.lock @@ -19,16 +19,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1614309161, - "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=", + "lastModified": 1622503062, + "narHash": "sha256-5QHhFydG1+ImYjeE7XLrPbCUZhWcSYrebXpYuY0XWz4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93", + "rev": "3a2e0c36e79cecaf196cbea23e75e74710140ea4", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-20.09-small", + "ref": "nixos-21.05-small", "type": "indirect" } }, diff --git a/flake.nix b/flake.nix index a445c7763..1bb05e6d7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; + inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: @@ -286,8 +286,8 @@ nativeBuildInputs = [ which ]; - configurePhase = - '' + configurePhase = '' + ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin From 9f1a7f9d370b17d760b5f97ae73118cabc099590 Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Tue, 1 Jun 2021 09:48:35 +0200 Subject: [PATCH 056/555] Include aarch64-darwin in installer Co-authored-by: Matthew Bauer --- flake.nix | 2 +- scripts/install.in | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index 1bb05e6d7..6c65e3406 100644 --- a/flake.nix +++ b/flake.nix @@ -387,7 +387,7 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. diff --git a/scripts/install.in b/scripts/install.in index f660e3807..39016d161 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -46,15 +46,9 @@ case "$(uname -s).$(uname -m)" in system=x86_64-darwin ;; Darwin.arm64|Darwin.aarch64) - # check for Rosetta 2 support - if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then - oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation" - fi - - hash=@tarballHash_x86_64-darwin@ - path=@tarballPath_x86_64-darwin@ - # eventually maybe: aarch64-darwin - system=x86_64-darwin + hash=@binaryTarball_aarch64-darwin@ + path=@tarballPath_aarch64-darwin@ + system=aarch64-darwin ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From c57ab1768767ca4f860ce5ca5faf54a7dba4969f Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 17 May 2021 17:08:57 +0000 Subject: [PATCH 057/555] Only link with libdl on Linux Linux is (as far as I know) the only mainstream operating system that requires linking with libdl for dlopen. On BSD, libdl doesn't exist, so on non-FreeBSD BSDs linking will currently fail. On macOS, it's apparently just a symlink to libSystem (macOS libc), presumably present for compatibility with things that assume Linux. So the right thing to do here is to only add -ldl on Linux, not to add it for everything that isn't FreeBSD. --- nix-rust/local.mk | 9 +++++++-- src/libexpr/local.mk | 2 +- src/libstore/local.mk | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 50db4783c..9650cdf93 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -8,8 +8,13 @@ endif libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) -libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust + +ifeq ($(OS), Linux) +libnixrust_LDFLAGS_USE += -ldl +libnixrust_LDFLAGS_USE_INSTALLED += -ldl +endif ifeq ($(OS), Darwin) libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 26c53d301..c40abfb78 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = -lboost_context -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libexpr_LDFLAGS += -ldl endif diff --git a/src/libstore/local.mk b/src/libstore/local.mk index cf0933705..b6652984c 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,7 +9,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libstore_LDFLAGS += -ldl endif From f357cea40ecfe47ead23fde3fcd59b8ced8252dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Jun 2021 11:42:38 +0200 Subject: [PATCH 058/555] Run autoupdate --- config/config.guess | 20 +++++++++++--------- config/config.sub | 20 +++++++++++++------- configure.ac | 27 ++++++++++----------------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/config/config.guess b/config/config.guess index 699b3a10b..1972fda8e 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-11-19' +timestamp='2021-01-25' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -188,10 +188,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ - "/sbin/$sysctl" 2>/dev/null || \ - "/usr/sbin/$sysctl" 2>/dev/null || \ + /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ echo unknown)) case "$UNAME_MACHINE_ARCH" in aarch64eb) machine=aarch64_be-unknown ;; @@ -996,6 +995,9 @@ EOF k1om:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; m32r*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; @@ -1084,7 +1086,7 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-"$LIBC" exit ;; - riscv32:Linux:*:* | riscv64:Linux:*:*) + riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) @@ -1480,8 +1482,8 @@ EOF i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos exit ;; - i*86:AROS:*:*) - echo "$UNAME_MACHINE"-pc-aros + *:AROS:*:*) + echo "$UNAME_MACHINE"-unknown-aros exit ;; x86_64:VMkernel:*:*) echo "$UNAME_MACHINE"-unknown-esx diff --git a/config/config.sub b/config/config.sub index 19c9553b1..63c1f1c8b 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-12-02' +timestamp='2021-01-08' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -1185,6 +1185,7 @@ case $cpu-$vendor in | k1om \ | le32 | le64 \ | lm32 \ + | loongarch32 | loongarch64 | loongarchx32 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ @@ -1229,7 +1230,7 @@ case $cpu-$vendor in | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ | pru \ | pyramid \ - | riscv | riscv32 | riscv64 \ + | riscv | riscv32 | riscv32be | riscv64 | riscv64be \ | rl78 | romp | rs6000 | rx \ | s390 | s390x \ | score \ @@ -1682,11 +1683,14 @@ fi # Now, validate our (potentially fixed-up) OS. case $os in - # Sometimes we do "kernel-abi", so those need to count as OSes. + # Sometimes we do "kernel-libc", so those need to count as OSes. musl* | newlib* | uclibc*) ;; - # Likewise for "kernel-libc" - eabi | eabihf | gnueabi | gnueabihf) + # Likewise for "kernel-abi" + eabi* | gnueabi*) + ;; + # VxWorks passes extra cpu info in the 4th filed. + simlinux | simwindows | spe) ;; # Now accept the basic system types. # The portable systems comes first. @@ -1750,6 +1754,8 @@ case $kernel-$os in ;; kfreebsd*-gnu* | kopensolaris*-gnu*) ;; + vxworks-simlinux | vxworks-simwindows | vxworks-spe) + ;; nto-qnx*) ;; os2-emx) diff --git a/configure.ac b/configure.ac index 05523eef8..3bce401db 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_INIT([nix],[m4_esyscmd(bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX")]) AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -9,8 +9,7 @@ AC_PROG_SED AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) -AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], - [Platform identifier (e.g., `i686-linux').]), +AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (e.g., `i686-linux').]), [system=$withval], [case "$host_cpu" in i*86) @@ -127,8 +126,7 @@ NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) -AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], - [path of the Nix store (defaults to /nix/store)]), +AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix store (defaults to /nix/store)]), storedir=$withval, storedir='/nix/store') AC_SUBST(storedir) @@ -157,8 +155,7 @@ fi PKG_PROG_PKG_CONFIG -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), +AC_ARG_ENABLE(shared, AS_HELP_STRING([--enable-shared],[Build shared libraries for Nix [default=yes]]), shared=$enableval, shared=yes) if test "$shared" = yes; then AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) @@ -215,9 +212,8 @@ AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then AC_ARG_ENABLE([seccomp-sandboxing], - AC_HELP_STRING([--disable-seccomp-sandboxing], - [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] - )) + AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!) + ])) if test "x$enable_seccomp_sandboxing" != "xno"; then PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) @@ -249,8 +245,7 @@ fi # Whether to use the Boehm garbage collector. -AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], - [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), +AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), gc=$enableval, gc=yes) if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) @@ -264,8 +259,7 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # documentation generation switch -AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], - [disable documentation generation]), +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) @@ -285,8 +279,7 @@ if test "$(uname)" = "Darwin"; then fi -AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], - [path of a statically-linked shell to use as /bin/sh in sandboxes]), +AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) @@ -301,6 +294,6 @@ done rm -f Makefile.config -AC_CONFIG_HEADER([config.h]) +AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([]) AC_OUTPUT From 5985b8b5275605ddd5e92e2f0a7a9f494ac6e35d Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 27 May 2021 13:25:25 +0200 Subject: [PATCH 059/555] Check the CA hash when importing stuff in the local store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When adding a path to the local store (via `LocalStore::addToStore`), ensure that the `ca` field of the provided `ValidPathInfo` does indeed correspond to the content of the path. Otherwise any untrusted user (or any binary cache) can add arbitrary content-addressed paths to the store (as content-addressed paths don’t need a signature). --- src/libstore/local-store.cc | 57 +++++++++++++++++++++++++++++ src/libstore/local-store.hh | 13 +++++++ tests/local.mk | 1 + tests/substitute-with-invalid-ca.sh | 38 +++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 tests/substitute-with-invalid-ca.sh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3b99ba39d..efea7bd23 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1168,6 +1168,31 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); + if (info.ca) { + if (auto foHash = std::get_if(&*info.ca)) { + auto actualFoHash = hashCAPath( + foHash->method, + foHash->hash.type, + info.path + ); + if (foHash->hash != actualFoHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + foHash->hash.to_string(Base32, true), + actualFoHash.hash.to_string(Base32, true)); + } + } + if (auto textHash = std::get_if(&*info.ca)) { + auto actualTextHash = hashString(htSHA256, readFile(realPath)); + if (textHash->hash != actualTextHash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + textHash->hash.to_string(Base32, true), + actualTextHash.to_string(Base32, true)); + } + } + } + autoGC(); canonicalisePathMetaData(realPath, -1); @@ -1672,4 +1697,36 @@ std::optional LocalStore::queryRealisation( .id = id, .outPath = outputPath, .signatures = signatures}}; }); } + + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, const HashType & hashType, + const StorePath & path) +{ + return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); +} + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash +) +{ + HashModuloSink caSink ( hashType, std::string(pathHash) ); + switch (method) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + auto hash = caSink.finish().first; + return FixedOutputHash{ + .method = method, + .hash = hash, + }; +} + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 26e034a82..c0b8e0da6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -283,6 +283,19 @@ private: void createUser(const std::string & userName, uid_t userId) override; + // XXX: Make a generic `Store` method + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const StorePath & path); + + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash + ); + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; diff --git a/tests/local.mk b/tests/local.mk index 542be6b7e..59eb4eb0f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -11,6 +11,7 @@ nix_tests = \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ + substitute-with-invalid-ca.sh \ binary-cache-build-remote.sh \ nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ diff --git a/tests/substitute-with-invalid-ca.sh b/tests/substitute-with-invalid-ca.sh new file mode 100644 index 000000000..4d0b01e0f --- /dev/null +++ b/tests/substitute-with-invalid-ca.sh @@ -0,0 +1,38 @@ +source common.sh + +BINARY_CACHE=file://$cacheDir + +getHash() { + basename "$1" | cut -d '-' -f 1 +} +getRemoteNarInfo () { + echo "$cacheDir/$(getHash "$1").narinfo" +} + +cat < $TEST_HOME/good.txt +I’m a good path +EOF + +cat < $TEST_HOME/bad.txt +I’m a bad path +EOF + +good=$(nix-store --add $TEST_HOME/good.txt) +bad=$(nix-store --add $TEST_HOME/bad.txt) +nix copy --to "$BINARY_CACHE" "$good" +nix copy --to "$BINARY_CACHE" "$bad" +nix-collect-garbage >/dev/null 2>&1 + +# Falsifying the narinfo file for '$good' +goodPathNarInfo=$(getRemoteNarInfo "$good") +badPathNarInfo=$(getRemoteNarInfo "$bad") +for fieldName in URL FileHash FileSize NarHash NarSize; do + sed -i "/^$fieldName/d" "$goodPathNarInfo" + grep -E "^$fieldName" "$badPathNarInfo" >> "$goodPathNarInfo" +done + +# Copying back '$good' from the binary cache. This should fail as it is +# corrupted +if nix copy --from "$BINARY_CACHE" "$good"; then + fail "Importing a path with a wrong CA field should fail" +fi From aedb5c7301d3ee45083d71137b228f4e79bf4868 Mon Sep 17 00:00:00 2001 From: Chua Hou Date: Wed, 2 Jun 2021 00:44:03 +0800 Subject: [PATCH 060/555] Install zsh completion script --- Makefile | 1 + misc/zsh/local.mk | 1 + 2 files changed, 2 insertions(+) create mode 100644 misc/zsh/local.mk diff --git a/Makefile b/Makefile index 68ec3ab0c..b7f0e79db 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ makefiles = \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ misc/bash/local.mk \ + misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ misc/upstart/local.mk \ diff --git a/misc/zsh/local.mk b/misc/zsh/local.mk new file mode 100644 index 000000000..418fb1377 --- /dev/null +++ b/misc/zsh/local.mk @@ -0,0 +1 @@ +$(eval $(call install-file-as, $(d)/completion.zsh, $(datarootdir)/zsh/site-functions/_nix, 0644)) From d12b12a15ba5cda49baacd22fbf7f9f526ed74e4 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Jun 2021 10:36:33 +0200 Subject: [PATCH 061/555] Let `nix flake check` keep going when keep-going is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the `keep-going` option is set to `true`, make `nix flake check` continue as much as it can before failing. The UI isn’t perfect as-it-is as all the lines currently start with a mostly useless `error (ignored): error:` prefix, but I’m not sure what the best output would be, so I’ll leave it as-it-is for the time being (This is a bit hijacking the `keep-going` flag as it’s supposed to be a build-time only thing. But I think it’s faire to reuse it here). Fix https://github.com/NixOS/nix/issues/4450 --- src/nix/flake-check.md | 2 ++ src/nix/flake.cc | 45 +++++++++++++++++++++++++++++------------- tests/flakes.sh | 15 ++++++++++++++ 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index ffe9d64b4..f7427d61d 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -22,6 +22,8 @@ This command verifies that the flake specified by flake reference that the derivations specified by the flake's `checks` output can be built successfully. +If the `keep-going` option is set to `true`, Nix will keep evaluating as much as it can and report the errors as it encounters them. Otherise it will stop at the first error. + # Evaluation checks The following flake output attributes must be derivations: diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 62a413e27..c2b4fb88e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -272,25 +272,40 @@ struct CmdFlakeCheck : FlakeCommand auto state = getEvalState(); auto flake = lockFlake(); + bool hasErrors = false; + auto throw_ = [&](const Error & e) { + try { + throw e; + } catch (Error & e) { + if (settings.keepGoing) { + ignoreException(); + hasErrors = true; + } + else + throw; + } + }; + // FIXME: rewrite to use EvalCache. auto checkSystemName = [&](const std::string & system, const Pos & pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) - throw Error("'%s' is not a valid system type, at %s", system, pos); + throw_(Error("'%s' is not a valid system type, at %s", system, pos)); }; - auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional { try { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); // FIXME: check meta attributes - return store->parseStorePath(drvInfo->queryDrvPath()); + return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); - throw; + throw_(e); } + return std::nullopt; }; std::vector drvPaths; @@ -307,7 +322,7 @@ struct CmdFlakeCheck : FlakeCommand #endif } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); - throw; + throw_(e); } }; @@ -323,7 +338,7 @@ struct CmdFlakeCheck : FlakeCommand // evaluate the overlay. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); - throw; + throw_(e); } }; @@ -347,7 +362,7 @@ struct CmdFlakeCheck : FlakeCommand // check the module. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); - throw; + throw_(e); } }; @@ -369,7 +384,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); - throw; + throw_(e); } }; @@ -384,7 +399,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); - throw; + throw_(e); } }; @@ -418,7 +433,7 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + throw_(e); } }; @@ -433,7 +448,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + throw_(e); } }; @@ -461,8 +476,8 @@ struct CmdFlakeCheck : FlakeCommand auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); - if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{drvPath}); + if (drvPath && (std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back(DerivedPath::Built{*drvPath}); } } } @@ -574,7 +589,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); - throw; + throw_(e); } }); } @@ -583,6 +598,8 @@ struct CmdFlakeCheck : FlakeCommand Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); store->buildPaths(drvPaths); } + if (hasErrors) + throw Error("Some errors were encountered during the evaluation"); } }; diff --git a/tests/flakes.sh b/tests/flakes.sh index e78e4a39d..9764e1a6c 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -535,6 +535,21 @@ EOF (! nix flake check $flake3Dir) +cat > $flake3Dir/flake.nix <&1 && fail "nix flake check should have failed" || true) +echo "$checkRes" | grep -q "defaultPackage.system-1" +echo "$checkRes" | grep -q "defaultPackage.system-2" + # Test 'follows' inputs. cat > $flake3Dir/flake.nix < Date: Wed, 2 Jun 2021 11:24:31 +0200 Subject: [PATCH 062/555] throw_ -> reportError This function might or might not throw depending on the value of `keepGoing`, so naming it `throw_` was a bit confusing --- src/nix/flake.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index c2b4fb88e..64fcfc000 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -273,7 +273,7 @@ struct CmdFlakeCheck : FlakeCommand auto flake = lockFlake(); bool hasErrors = false; - auto throw_ = [&](const Error & e) { + auto reportError = [&](const Error & e) { try { throw e; } catch (Error & e) { @@ -291,7 +291,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkSystemName = [&](const std::string & system, const Pos & pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) - throw_(Error("'%s' is not a valid system type, at %s", system, pos)); + reportError(Error("'%s' is not a valid system type, at %s", system, pos)); }; auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional { @@ -303,7 +303,7 @@ struct CmdFlakeCheck : FlakeCommand return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); - throw_(e); + reportError(e); } return std::nullopt; }; @@ -322,7 +322,7 @@ struct CmdFlakeCheck : FlakeCommand #endif } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -338,7 +338,7 @@ struct CmdFlakeCheck : FlakeCommand // evaluate the overlay. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -362,7 +362,7 @@ struct CmdFlakeCheck : FlakeCommand // check the module. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -384,7 +384,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -399,7 +399,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -433,7 +433,7 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -448,7 +448,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw_(e); + reportError(e); } }; @@ -589,7 +589,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); - throw_(e); + reportError(e); } }); } From 7565308d0474f5f76758d0ca2487d63e5b6335c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 2 Jun 2021 11:25:47 +0200 Subject: [PATCH 063/555] Fix a documentation typo Co-authored-by: Eelco Dolstra --- src/nix/flake-check.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index f7427d61d..12d33c5f9 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -22,7 +22,7 @@ This command verifies that the flake specified by flake reference that the derivations specified by the flake's `checks` output can be built successfully. -If the `keep-going` option is set to `true`, Nix will keep evaluating as much as it can and report the errors as it encounters them. Otherise it will stop at the first error. +If the `keep-going` option is set to `true`, Nix will keep evaluating as much as it can and report the errors as it encounters them. Otherwise it will stop at the first error. # Evaluation checks From 838f862f4f89346fa8510317b5b1505633c28ff6 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 2 Jun 2021 11:26:04 +0200 Subject: [PATCH 064/555] doc: Wrap at 80 characters --- src/nix/flake-check.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index 12d33c5f9..8ef932954 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -22,7 +22,9 @@ This command verifies that the flake specified by flake reference that the derivations specified by the flake's `checks` output can be built successfully. -If the `keep-going` option is set to `true`, Nix will keep evaluating as much as it can and report the errors as it encounters them. Otherwise it will stop at the first error. +If the `keep-going` option is set to `true`, Nix will keep evaluating as much +as it can and report the errors as it encounters them. Otherwise it will stop +at the first error. # Evaluation checks From 7c3cb8506fb3046cb97e1b98f3b1195d45c500f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 2 Jun 2021 10:50:47 +0200 Subject: [PATCH 065/555] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3a2e0c36e79cecaf196cbea23e75e74710140ea4' -> 'github:NixOS/nixpkgs/bb8a5e54845012ed1375ffd5f317d2fdf434b20e' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 325e01fb8..8aad22957 100644 --- a/flake.lock +++ b/flake.lock @@ -19,11 +19,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1622503062, - "narHash": "sha256-5QHhFydG1+ImYjeE7XLrPbCUZhWcSYrebXpYuY0XWz4=", + "lastModified": 1622593737, + "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a2e0c36e79cecaf196cbea23e75e74710140ea4", + "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e", "type": "github" }, "original": { From 50dc88a56cc13030801bfefc7116828d4115ae70 Mon Sep 17 00:00:00 2001 From: keke Date: Wed, 2 Jun 2021 18:09:03 +0800 Subject: [PATCH 066/555] fix error: 'optional' in namespace 'std' does not name a template type --- src/libfetchers/attrs.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index a2d53a7bf..e41037633 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -6,6 +6,8 @@ #include +#include + namespace nix::fetchers { typedef std::variant> Attr; From f35f9af78720cec098c5c2188bf34948b3a67c95 Mon Sep 17 00:00:00 2001 From: Keshav Kini Date: Thu, 3 Jun 2021 17:59:39 -0700 Subject: [PATCH 067/555] Improve explanation of NIX_PATH prefix syntax The previous wording seemed to imply that the "channel:" syntax would resolve to a github archive URL, which is not the case. --- doc/manual/src/command-ref/env-common.md | 42 +++++++++++++----------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b709ca9d1..6e2403461 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables: A colon-separated list of directories used to look up Nix expressions enclosed in angle brackets (i.e., ``). For instance, the value - + /home/eelco/Dev:/etc/nixos - + will cause Nix to look for paths relative to `/home/eelco/Dev` and `/etc/nixos`, in this order. It is also possible to match paths against a prefix. For example, the value - + nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos - + will cause Nix to search for `` in `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - + If a path in the Nix search path starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. The tarball must consist of a single top-level directory. For example, setting `NIX_PATH` to - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz - - tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09 - channel. - - A following shorthand can be used to refer to the official channels: - - nixpkgs=channel:nixos-15.09 - - The search path can be extended using the `-I` option, which takes - precedence over `NIX_PATH`. + + nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + + tells Nix to download and use the current contents of the + `master` branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official nixos.org channels (see + [the manual for `nix-channel`](nix-channel.md)) can be abbreviated + as `channel:`. For instance, the following two + values of `NIX_PATH` are equivalent: + + nixpkgs=channel:nixos-21.05 + nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + + The Nix search path can also be extended using the `-I` option to + many Nix commands, which takes precedence over `NIX_PATH`. - `NIX_IGNORE_SYMLINK_STORE`\ Normally, the Nix store directory (typically `/nix/store`) is not @@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables: builds are deployed to machines where `/nix/store` resolves differently. If you are sure that you’re not going to do that, you can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - + Note that if you’re symlinking the Nix store so that you can put it on another file system than the root file system, on Linux you’re better off using `bind` mount points, e.g., @@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables: $ mkdir /nix $ mount -o bind /mnt/otherdisk/nix /nix ``` - + Consult the mount 8 manual page for details. - `NIX_STORE_DIR`\ From 196b77b686ab407e4e01fb3ecde07a5f199f4045 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 4 Jun 2021 11:25:36 +0000 Subject: [PATCH 068/555] configure.ac: fix use of unread LIBS variable This fixes both the SunOS/Solaris check, and the libatomic check, which reference $LIBS, which has not been used since automake was stripped out of the code. --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 3bce401db..03b123366 100644 --- a/configure.ac +++ b/configure.ac @@ -65,7 +65,7 @@ AC_SYS_LARGEFILE AC_STRUCT_DIRENT_D_TYPE if test "$sys_name" = sunos; then # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" + LDFLAGS="-lsocket -lnsl $LDFLAGS" fi @@ -150,7 +150,7 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="-latomic $LDFLAGS" fi PKG_PROG_PKG_CONFIG From 7c077d2a0f4ccdeb84a4ad14ba5871a3fbe170e4 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Jun 2021 08:37:59 +0200 Subject: [PATCH 069/555] Add a ca-derivations required machine feature Make ca-derivations require a `ca-derivations` machine feature, and ca-aware builders expose it. That way, a network of builders can mix ca-aware and non-ca-aware machines, and the scheduler will send them in the right place. --- src/libstore/parsed-derivations.cc | 2 ++ src/libstore/store-api.cc | 7 +++++++ src/libstore/store-api.hh | 4 +++- tests/build-remote-content-addressed-floating.sh | 2 ++ tests/build-remote.sh | 11 ++++++++--- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c5c3ae3dc..5e383a9a4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -91,6 +91,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); + if (!derivationHasKnownOutputPaths(drv.type())) + res.insert("ca-derivations"); return res; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 93fcb068f..6ca8e5b24 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -337,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, return info; } +StringSet StoreConfig::getDefaultSystemFeatures() +{ + auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + res.insert("ca-derivations"); + return res; +} Store::Store(const Params & params) : StoreConfig(params) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f66298991..9b4b10727 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -180,6 +180,8 @@ struct StoreConfig : public Config StoreConfig() = delete; + StringSet getDefaultSystemFeatures(); + virtual ~StoreConfig() { } virtual const std::string name() = 0; @@ -196,7 +198,7 @@ struct StoreConfig : public Config Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; - Setting systemFeatures{this, settings.systemFeatures, + Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index 7447d92bd..13ef47d2d 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -4,4 +4,6 @@ file=build-hook-ca-floating.nix sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf +CONTENT_ADDRESSED=true + source build-remote.sh diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 70f82e939..27d85a83d 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -6,12 +6,17 @@ unset NIX_STATE_DIR function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } +EXTRA_SYSTEM_FEATURES=() +if [[ -n "$CONTENT_ADDRESSED" ]]; then + EXTRA_SYSTEM_FEATURES=("ca-derivations") +fi + builders=( # system-features will automatically be added to the outer URL, but not inner # remote-store URL. - "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo" - "$TEST_ROOT/machine2 - - 1 1 bar" - "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz" + "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})" + "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})" + "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})" ) chmod -R +w $TEST_ROOT/machine* || true From 7ac038fa4b83aaeff0b4dcf97b912cd7fd5f0ef6 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Jun 2021 09:24:24 +0200 Subject: [PATCH 070/555] Make `computeFSClosure` ca-aware Fix #4820 by preventing nix-collect garbage from crashing if `keep-outputs` or `keep-derivations` is true --- src/libstore/misc.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bc5fd968c..3e54bfd8f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto& i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - res.insert(i); + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == path) + res.insert(*maybeOutPath); return res; }; else @@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto& i : queryDerivationOutputs(path)) - if (isValidPath(i)) - res.insert(i); + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); if (includeDerivers && info->deriver && isValidPath(*info->deriver)) res.insert(*info->deriver); From 56605b468868b834e44a9700907b734428cb120a Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 20 May 2021 13:55:00 +0200 Subject: [PATCH 071/555] Make `nix-shell` support content-addressed derivations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve the derivation before trying to load its environment − essentially reproducing what the build loop does − so that we can effectively access our dependencies (and not just their placeholders). Fix #4821 --- src/nix-build/nix-build.cc | 7 +++++++ tests/ca/nix-shell.sh | 10 ++++++++++ tests/local.mk | 1 + tests/nix-shell.sh | 14 ++++++++++---- tests/shell.nix | 16 ++++++++++++++-- 5 files changed, 42 insertions(+), 6 deletions(-) create mode 100755 tests/ca/nix-shell.sh diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9acbedda2..05eb7e234 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -387,6 +387,13 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = drv.tryResolve(*store); + if (!resolvedDrv) + throw Error("unable to resolve the derivation '%s'. nix-shell can’t continue", drvInfo.queryDrvPath()); + drv = *resolvedDrv; + } + // Set the environment. auto env = getEnv(); diff --git a/tests/ca/nix-shell.sh b/tests/ca/nix-shell.sh new file mode 100755 index 000000000..7f1a3a73e --- /dev/null +++ b/tests/ca/nix-shell.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf + +CONTENT_ADDRESSED=true +cd .. +source ./nix-shell.sh + diff --git a/tests/local.mk b/tests/local.mk index 59eb4eb0f..d640bdd3c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -47,6 +47,7 @@ nix_tests = \ ca/build.sh \ ca/substitute.sh \ ca/signatures.sh \ + ca/nix-shell.sh \ ca/nix-run.sh \ ca/nix-copy.sh # parallel.sh diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 4775bafb9..e3174dac1 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -2,6 +2,12 @@ source common.sh clearStore +if [[ -n ${CONTENT_ADDRESSED:-} ]]; then + nix-shell () { + command nix-shell --arg contentAddressed true "$@" + } +fi + # Test nix-shell -A export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz @@ -41,7 +47,7 @@ output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(f [ "$output" = "foo bar" ] # Test nix-shell shebang mode -sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) @@ -49,7 +55,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def) # Test nix-shell shebang mode again with metacharacters in the filename. # First word of filename is chosen to not match any file in the test root. -sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) @@ -58,7 +64,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) # Test nix-shell shebang mode for ruby # This uses a fake interpreter that returns the arguments passed # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. -sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb chmod a+rx $TEST_ROOT/shell.shebang.rb output=$($TEST_ROOT/shell.shebang.rb abc ruby) @@ -66,7 +72,7 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby) # Test nix-shell shebang mode for ruby again with metacharacters in the filename. # Note: fake interpreter only space-separates args without adding escapes to its output. -sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) diff --git a/tests/shell.nix b/tests/shell.nix index 24ebcc04c..53759f99a 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -1,6 +1,18 @@ -{ inNixShell ? false }: +{ inNixShell ? false, contentAddressed ? false }: -with import ./config.nix; +let cfg = import ./config.nix; in +with cfg; + +let + mkDerivation = + if contentAddressed then + args: cfg.mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // args) + else cfg.mkDerivation; +in let pkgs = rec { setupSh = builtins.toFile "setup" '' From 2cf591a134f3ec6f634b47eeb522f422c64a9d33 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Jun 2021 13:31:19 +0200 Subject: [PATCH 072/555] Make `nix develop` work with CA derivations Fix #4823 --- src/nix/develop.cc | 30 +++++++++++++++++++----------- tests/nix-shell.sh | 12 ++++++++++-- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 10f843651..aeeaf3e13 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -144,17 +144,26 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; - for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; - drv.env[output.first] = ""; - } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = ""; + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - for (auto & output : drv.outputs) { - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; - drv.env[output.first] = store->printStorePath(outPath); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } } auto shellDrvPath = writeDerivation(*store, drv); @@ -162,8 +171,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Build the derivation. */ store->buildPaths({DerivedPath::Built{shellDrvPath}}); - for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { - auto & [_1, optPath] = outputAndOptPath; + for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index e3174dac1..3481c2c69 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -6,6 +6,14 @@ if [[ -n ${CONTENT_ADDRESSED:-} ]]; then nix-shell () { command nix-shell --arg contentAddressed true "$@" } + + nix_develop() { + nix develop --arg contentAddressed true "$@" + } +else + nix_develop() { + nix develop "$@" + } fi # Test nix-shell -A @@ -79,13 +87,13 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. -nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' +nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' # Ensure `nix develop -c` preserves stdin echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal -nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo +nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) From 96d7170e12c75acf363bf7a5a081a86b3c384384 Mon Sep 17 00:00:00 2001 From: regnat Date: Sat, 12 Jun 2021 12:24:53 +0200 Subject: [PATCH 073/555] =?UTF-8?q?Don=E2=80=99t=20check=20the=20`deriver`?= =?UTF-8?q?=20field=20on=20computeFSClosure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That doesn’t really make sense with CA derivations (and wasn’t even really correct before because of FO derivations, though that probably didn’t matter much in practice) --- src/libstore/misc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 3e54bfd8f..48442563f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -30,7 +30,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, if (includeDerivers && path.isDerivation()) for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) - if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == path) + if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; }; From 79674c6cdb63a2e9421e38e47d5fe5b5c5ffbb32 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 12 Jun 2021 23:50:26 -0500 Subject: [PATCH 074/555] Fix zsh completion script Installed site-functions need to be run directly, not via compdef. --- misc/zsh/completion.zsh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index d4df6447e..a902e37dc 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -1,3 +1,5 @@ +#compdef nix + function _nix() { local ifs_bk="$IFS" local input=("${(Q)words[@]}") @@ -18,4 +20,4 @@ function _nix() { _describe 'nix' suggestions } -compdef _nix nix +_nix "$@" From e6150de90d8db101209fc6363f5f7696ee8192c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jun 2021 12:06:01 +0200 Subject: [PATCH 075/555] nix develop: Filter out NIX_REMOTE When recursive Nix is enabled, NIX_REMOTE is set to unix:///build/.nix-socket, which doesn't work outside of the sandbox. --- src/nix/develop.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 10f843651..e51de2de8 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -185,6 +185,7 @@ struct Common : InstallableCommand, MixProfile "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", + "NIX_REMOTE", "PPID", "PWD", "SHELLOPTS", From a3ce88725b44a30184cbe8706fc3a53a8610cc47 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Jun 2021 12:11:31 +0200 Subject: [PATCH 076/555] Add a test for the gc with CA derivations Also add a small architecture to easily run CA-enabled tests --- tests/ca/gc.sh | 12 ++++++++++++ tests/config.nix.in | 11 ++++++++++- tests/local.mk | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 tests/ca/gc.sh diff --git a/tests/ca/gc.sh b/tests/ca/gc.sh new file mode 100755 index 000000000..e4f9857d6 --- /dev/null +++ b/tests/ca/gc.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Ensure that garbage collection works properly with ca derivations + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source gc.sh diff --git a/tests/config.nix.in b/tests/config.nix.in index a57a8c596..9b00d9ddb 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -1,3 +1,12 @@ +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; +in + rec { shell = "@bash@"; @@ -15,4 +24,4 @@ rec { PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; -} +} // caArgs diff --git a/tests/local.mk b/tests/local.mk index 59eb4eb0f..e40178b80 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -2,6 +2,7 @@ nix_tests = \ hash.sh lang.sh add.sh simple.sh dependencies.sh \ config.sh \ gc.sh \ + ca/gc.sh \ gc-concurrent.sh \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ From 769ca4e26d973660756de85b51efdb7fc36a00fd Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Thu, 17 Jun 2021 12:00:26 -0600 Subject: [PATCH 077/555] libfetchers/git: fetch submodules by default --- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/git.cc | 2 +- tests/fetchGitSubmodules.sh | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b8b99d4fa..9b0c4914b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -44,7 +44,7 @@ void emitTreeAttrs( if (input.getType() == "git") mkBool(*state.allocAttr(v, state.symbols.create("submodules")), - fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); + fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(true)); if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d8e0dbe0a..4ce6c5b7a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -172,7 +172,7 @@ struct GitInputScheme : InputScheme Input input(_input); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(true); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); std::string cacheType = "git"; diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 5f104355f..03d46088e 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -42,8 +42,8 @@ r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \ r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -[[ $r1 == $r2 ]] -[[ $r2 != $r3 ]] +[[ $r1 == $r3 ]] +[[ $r2 != $r1 ]] r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") @@ -52,13 +52,13 @@ r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $r1 == $r4 ]] -[[ $r4 == $r5 ]] +[[ $r4 == $r6 ]] [[ $r3 == $r6 ]] [[ $r6 == $r7 ]] [[ $r7 == $r8 ]] have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") -[[ $have_submodules == false ]] +[[ $have_submodules == true ]] have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") [[ $have_submodules == false ]] @@ -66,8 +66,8 @@ have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \ have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] -pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") From 4202a3bc4e7ce687be245539f6d796226f7d9e37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Jun 2021 17:04:11 +0200 Subject: [PATCH 078/555] UDSRemoteStore: Support the 'root' store parameter Useful when we're using a daemon with a chroot store, e.g. $ NIX_DAEMON_SOCKET_PATH=/tmp/chroot/nix/var/nix/daemon-socket/socket nix-daemon --store /tmp/chroot Then the client can now connect with $ nix build --store unix:///tmp/chroot/nix/var/nix/daemon-socket/socket?root=/tmp/chroot nixpkgs#hello --- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/libstore/gc.cc | 4 ++-- src/libstore/local-fs-store.hh | 5 ++++- src/libstore/local-store.cc | 13 +++++-------- src/libstore/local-store.hh | 5 ----- src/libstore/optimise-store.cc | 2 +- src/nix/run.cc | 4 ++-- 7 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 28981a1a1..73e2350ea 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -292,7 +292,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -417,7 +417,7 @@ void LocalDerivationGoal::startBuilder() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir) { + if (localStore.storeDir != localStore.realStoreDir.get()) { #if __linux__ useChroot = true; #else diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index bc692ca42..5a62c6529 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { - AutoCloseDir dir(opendir(realStoreDir.c_str())); + AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that @@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.c_str(), &st)) + if (statvfs(realStoreDir.get().c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 55941b771..f8b19d00d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; + const PathSetting realStoreDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}; }; class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store @@ -34,7 +37,7 @@ public: /* Register a permanent GC root. */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); - virtual Path getRealStoreDir() { return storeDir; } + virtual Path getRealStoreDir() { return realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index efea7bd23..28c175a51 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -106,9 +106,6 @@ LocalStore::LocalStore(const Params & params) , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) - , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"} - , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") @@ -153,13 +150,13 @@ LocalStore::LocalStore(const Params & params) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; - if (stat(realStoreDir.c_str(), &st)) + if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) + if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.c_str(), perm) == -1) + if (chmod(realStoreDir.get().c_str(), perm) == -1) throw SysError("changing permissions on path '%1%'", realStoreDir); } } @@ -437,14 +434,14 @@ void LocalStore::makeStoreWritable() if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) + if (statvfs(realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } #endif diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c0b8e0da6..15c7fc306 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -83,9 +83,6 @@ private: public: - PathSetting realStoreDir_; - - const Path realStoreDir; const Path dbDir; const Path linksDir; const Path reservedPath; @@ -279,8 +276,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - Path getRealStoreDir() override { return realStoreDir; } - void createUser(const std::string & userName, uid_t userId) override; // XXX: Make a generic `Store` method diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 78d587139..d95e54af1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ - bool mustToggle = dirOf(path) != realStoreDir; + bool mustToggle = dirOf(path) != realStoreDir.get(); if (mustToggle) makeWritable(dirOf(path)); /* When we're done, make the directory read-only again and reset diff --git a/src/nix/run.cc b/src/nix/run.cc index f684c5ea4..c0ba05a3e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -43,8 +43,8 @@ struct RunCommon : virtual Command helper program (chrootHelper() below) to do the work. */ auto store2 = store.dynamic_pointer_cast(); - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + if (store2 && store->storeDir != store2->getRealStoreDir()) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); From 498677cbedf9c7dee1df86e775342cdf93a110bd Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Jun 2021 14:16:13 +0200 Subject: [PATCH 079/555] Fix the remote build of CA derivations Make sure that the derivation we send to the remote builder is exactly the one that we want to build locally so that the output ids are exactly the same Fix #4845 --- src/build-remote/build-remote.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 57f2cd32d..0904f2ce9 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -277,7 +277,16 @@ connected: auto drv = store->readDerivation(*drvPath); auto outputHashes = staticOutputHashes(*store, drv); - drv.inputSrcs = store->parseStorePathSet(inputs); + + // Hijack the inputs paths of the derivation to include all the paths + // that come from the `inputDrvs` set. + // We don’t do that for the derivations whose `inputDrvs` is empty + // because + // 1. It’s not needed + // 2. Changing the `inputSrcs` set changes the associated output ids, + // which break CA derivations + if (!drv.inputDrvs.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); From 3784c66a46aea2cc3e26b8cff94b3a1f477888f1 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Jun 2021 15:17:31 +0200 Subject: [PATCH 080/555] Remove a possible existing store path when building CA derivations In case a previous interrupted build left a garbage path laying around, remove it before trying to move the path to its final location. Fix #4858 --- src/libstore/build/local-derivation-goal.cc | 1 + tests/ca/build-with-garbage-path.sh | 20 ++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 22 insertions(+) create mode 100755 tests/ca/build-with-garbage-path.sh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 73e2350ea..279139020 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2487,6 +2487,7 @@ void LocalDerivationGoal::registerOutputs() assert(newInfo.ca); } else { auto destPath = worker.store.toRealPath(finalDestPath); + deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; } diff --git a/tests/ca/build-with-garbage-path.sh b/tests/ca/build-with-garbage-path.sh new file mode 100755 index 000000000..e6f878702 --- /dev/null +++ b/tests/ca/build-with-garbage-path.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Regression test for https://github.com/NixOS/nix/issues/4858 + +source common.sh +sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf + +# Get the output path of `rootCA`, and put some garbage instead +outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)" +nix-store --delete "$outPath" +touch "$outPath" + +# The build should correctly remove the garbage and put the expected path instead +nix-build ./content-addressed.nix -A rootCA --no-out-link + +# Rebuild it. This shouldn’t overwrite the existing path +oldInode=$(stat -c '%i' "$outPath") +nix-build ./content-addressed.nix -A rootCA --no-out-link --arg seed 2 +newInode=$(stat -c '%i' "$outPath") +[[ "$oldInode" == "$newInode" ]] diff --git a/tests/local.mk b/tests/local.mk index 59eb4eb0f..663b8c8bc 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -45,6 +45,7 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ + ca/build-with-garbage-path.sh \ ca/substitute.sh \ ca/signatures.sh \ ca/nix-run.sh \ From 608434722b1f6a25bd4dba6b28e7f7e516c687fb Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Jun 2021 15:47:47 +0200 Subject: [PATCH 081/555] Only symlink the requested outputs in `nix build` Fix #4925 --- src/nix/build.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index d924fe553..15923ebc3 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -69,8 +69,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(bo.path, absPath(symlink)); }, [&](BuiltPath::Built bfd) { - auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); - for (auto & output : builtOutputs) { + for (auto & output : bfd.outputs) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); From ce674cb2cf97fc14ce48acef7f8ef88f4ac771f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Jun 2021 15:52:01 +0200 Subject: [PATCH 082/555] Properly set the output env variables Co-authored-by: John Ericson --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index aeeaf3e13..350b08648 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -150,7 +150,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) output.second = { .output = DerivationOutputDeferred{}, }; - drv.env[output.first] = ""; + drv.env[output.first] = hashPlaceholder(output.first); } } else { for (auto & output : drv.outputs) { From dcabb461242e5c38e0a2780dc5869b91c13241d6 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Jun 2021 16:28:06 +0200 Subject: [PATCH 083/555] Shorten a stupidly long sql query name --- src/libstore/local-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index debe70037..45bb5efc8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -59,8 +59,8 @@ struct LocalStore::State::Stmts { SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; - SQLiteStmt QueryRealisationRealisationReferences; - SQLiteStmt AddRealisationRealisationReference; + SQLiteStmt QueryRealisationReferences; + SQLiteStmt AddRealisationReference; }; int getSchema(Path schemaPath) @@ -362,13 +362,13 @@ LocalStore::LocalStore(const Params & params) where drvPath = ? ; )"); - state->stmts->QueryRealisationRealisationReferences.create(state->db, + state->stmts->QueryRealisationReferences.create(state->db, R"( select drvPath, outputName from Realisations join RealisationsRefs on realisationReference = Realisations.id where referrer = ?; )"); - state->stmts->AddRealisationRealisationReference.create(state->db, + state->stmts->AddRealisationReference.create(state->db, R"( insert or replace into RealisationsRefs (referrer, realisationReference) values ( @@ -719,7 +719,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) .exec(); uint64_t myId = state->db.getLastInsertedRowId(); for (auto & [outputId, _] : info.dependentRealisations) { - state->stmts->AddRealisationRealisationReference + state->stmts->AddRealisationReference .use()(myId)(outputId.strHash())(outputId.outputName) .exec(); } @@ -1732,7 +1732,7 @@ std::optional LocalStore::queryRealisation( std::map dependentRealisations; auto useRealisationRefs( - state->stmts->QueryRealisationRealisationReferences.use()( + state->stmts->QueryRealisationReferences.use()( realisationDbId)); while (useRealisationRefs.next()) { auto depHash = useRealisationRefs.getStr(0); From 9676c9f6a31656beae6dc65e79ab2f621ccc1f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 21 Jun 2021 19:40:51 +0200 Subject: [PATCH 084/555] perlBindings: fix build on aarch64-darwin --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 6c65e3406..466145366 100644 --- a/flake.nix +++ b/flake.nix @@ -256,7 +256,8 @@ boost nlohmann_json ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} From 3b58dbb3561e02dc94e88b531d0942a7f8ccf8ea Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Jun 2021 11:29:55 +0200 Subject: [PATCH 085/555] nix-shell: Replace resolving failure error by an assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This shouldn’t happen in practice, so better make it explicit --- src/nix-build/nix-build.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 05eb7e234..3fec2c06c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -389,8 +389,7 @@ static void main_nix_build(int argc, char * * argv) if (settings.isExperimentalFeatureEnabled("ca-derivations")) { auto resolvedDrv = drv.tryResolve(*store); - if (!resolvedDrv) - throw Error("unable to resolve the derivation '%s'. nix-shell can’t continue", drvInfo.queryDrvPath()); + assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; } From 3b5429aec110a22f8fe0d92b72faf1921b8ede2d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 2 May 2021 17:24:14 +0200 Subject: [PATCH 086/555] Source complete env in `nix-shell` with `__structuredAttrs = true;` This is needed to push the adoption of structured attrs[1] forward. It's now checked if a `__json` exists in the environment-map of the derivation to be openend in a `nix-shell`. Derivations with structured attributes enabled also make use of a file named `.attrs.json` containing every environment variable represented as JSON which is useful for e.g. `exportReferencesGraph`[2]. To provide an environment similar to the build sandbox, `nix-shell` now adds a `.attrs.json` to `cwd` (which is mostly equal to the one in the build sandbox) and removes it using an exit hook when closing the shell. To avoid leaking internals of the build-process to the `nix-shell`, the entire logic to generate JSON and shell code for structured attrs was moved into the `ParsedDerivation` class. [1] https://nixos.mayflower.consulting/blog/2020/01/20/structured-attrs/ [2] https://nixos.org/manual/nix/unstable/expressions/advanced-attributes.html#advanced-attributes --- src/libstore/build/derivation-goal.cc | 37 ------- src/libstore/build/local-derivation-goal.cc | 113 ++------------------ src/libstore/parsed-derivations.cc | 110 +++++++++++++++++++ src/libstore/parsed-derivations.hh | 4 + src/libstore/store-api.cc | 36 +++++++ src/libstore/store-api.hh | 2 + src/nix-build/nix-build.cc | 45 +++++++- tests/structured-attrs-shell.nix | 19 ++++ tests/structured-attrs.sh | 6 ++ 9 files changed, 227 insertions(+), 145 deletions(-) create mode 100644 tests/structured-attrs-shell.nix diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 9100d3333..4c49f3cab 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -143,7 +143,6 @@ void DerivationGoal::work() (this->*state)(); } - void DerivationGoal::addWantedOutputs(const StringSet & outputs) { /* If we already want all outputs, there is nothing to do. */ @@ -1048,42 +1047,6 @@ HookReply DerivationGoal::tryBuildHook() } -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 279139020..9bb6f276c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -518,7 +518,7 @@ void LocalDerivationGoal::startBuilder() /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + worker.store.exportReferences({storePath}, inputPaths), false, false)); } } @@ -1084,113 +1084,18 @@ void LocalDerivationGoal::initEnv() } -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - void LocalDerivationGoal::writeStructuredAttrs() { - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; + if (auto structAttrs = parsedDrv->generateStructuredAttrs(inputRewrites, worker.store, inputPaths)) { + auto value = structAttrs.value(); + auto jsonSh = value.first; + auto json = value.second; - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c5c3ae3dc..029da8bd3 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,6 +1,8 @@ #include "parsed-derivations.hh" #include +#include +#include "json.hh" namespace nix { @@ -121,4 +123,112 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +std::optional ParsedDerivation::generateStructuredAttrs( + std::optional inputRewrites, Store & store, const StorePathSet & inputPaths) +{ + auto structuredAttrs = getStructuredAttrs(); + if (!structuredAttrs) return std::nullopt; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv.outputs) { + if (inputRewrites) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites.value()); + } else { + /* This case is only relevant for the nix-shell */ + outputs[i.first] = hashPlaceholder(i.first); + } + } + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get())); + store.pathInfoToJSON(jsonRoot, + store.exportReferences(storePaths, inputPaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + return std::make_pair(jsonSh, json); +} } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index c9fbe68c4..4b8b8c8ff 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -6,6 +6,8 @@ namespace nix { +typedef std::pair StructuredAttrsWithShellRC; + class ParsedDerivation { StorePath drvPath; @@ -36,6 +38,8 @@ public: bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; + + std::optional generateStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths); }; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 93fcb068f..cfdfb5269 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -627,6 +627,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths, } +StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths) +{ + StorePathSet paths; + + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath)); + + computeFSClosure({storePath}, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = paths; + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = derivationFromPath(j); + for (auto & k : drv.outputsAndOptPaths(*this)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + computeFSClosure(*k.second.second, paths); + } + } + } + + return paths; +} + + void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f66298991..b742fed6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -695,6 +695,8 @@ public: const Stats & getStats(); + StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr getBuildLog(const StorePath & path) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9acbedda2..f91a56d6a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -1,10 +1,15 @@ #include #include #include +#include #include #include #include +#include +#include + +#include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -422,12 +427,41 @@ static void main_nix_build(int argc, char * * argv) } else env[var.first] = var.second; + std::string structuredAttrsRC; + std::string exitCmd; + + if (env.count("__json")) { + StorePathSet inputs; + for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { + auto outputs = store->queryPartialDerivationOutputMap(depDrvPath); + for (auto & i : wantedDepOutputs) { + auto o = outputs.at(i); + store->computeFSClosure(*o, inputs); + } + } + + ParsedDerivation parsedDrv( + StorePath(store->parseStorePath(drvInfo.queryDrvPath())), + drv + ); + + if (auto structAttrs = parsedDrv.generateStructuredAttrs(std::nullopt, *store, inputs)) { + auto val = structAttrs.value(); + structuredAttrsRC = val.first; + auto attrsJSON = std::filesystem::current_path().string() + "/.attrs.json"; + writeFile(attrsJSON, val.second.dump()); + exitCmd = "\n_rm_attrs_json() { rm -f " + attrsJSON + "; }" + + "\nexitHooks+=(_rm_attrs_json)" + + "\nfailureHooks+=(_rm_attrs_json)\n"; + } + } + /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't lose the current $PATH directories. */ auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, fmt( + std::string rc = fmt( R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s + (keepTmp ? "trap _nix_shell_clean_tmpdir EXIT; " @@ -436,8 +470,9 @@ static void main_nix_build(int argc, char * * argv) "_nix_shell_clean_tmpdir; ") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" - "dontAddDisableDepTrack=1; " - "[ -e $stdenv/setup ] && source $stdenv/setup; " + "dontAddDisableDepTrack=1;\n" + + structuredAttrsRC + exitCmd + + "\n[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "PATH=%4%:\"$PATH\"; " "SHELL=%5%; " @@ -455,7 +490,9 @@ static void main_nix_build(int argc, char * * argv) shellEscape(dirOf(*shell)), shellEscape(*shell), (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), - envCommand)); + envCommand); + vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); + writeFile(rcfile, rc); Strings envStrs; for (auto & i : env) diff --git a/tests/structured-attrs-shell.nix b/tests/structured-attrs-shell.nix new file mode 100644 index 000000000..0c01c2568 --- /dev/null +++ b/tests/structured-attrs-shell.nix @@ -0,0 +1,19 @@ +with import ./config.nix; +let + dep = mkDerivation { + name = "dep"; + buildCommand = '' + mkdir $out; echo bla > $out/bla + ''; + }; +in +mkDerivation { + name = "structured2"; + __structuredAttrs = true; + outputs = [ "out" "dev" ]; + my.list = [ "a" "b" "c" ]; + exportReferencesGraph.refs = [ dep ]; + buildCommand = '' + touch ''${outputs[out]}; touch ''${outputs[dev]} + ''; +} diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index dcfe6d580..c0fcb021a 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -8,3 +8,9 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result [[ $(cat $TEST_ROOT/result/foo) = bar ]] [[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] + +export NIX_BUILD_SHELL=$SHELL +[[ ! -e '.attrs.json' ]] +env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ + --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < .attrs.json)"' +[[ ! -e '.attrs.json' ]] From 3944a120ec6986c723bf36bfade9b331dd4af68a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 9 May 2021 17:46:16 +0200 Subject: [PATCH 087/555] Set environment variables for .attrs.json & .attrs.sh This way no derivation has to expect that these files are in the `cwd` during the build. This is problematic for `nix-shell` where these files would have to be inserted into the nix-shell's `cwd` which can become problematic with e.g. recursive `nix-shell`. To remain backwards-compatible, the location inside the build sandbox will be kept, however using these files directly should be deprecated from now on. --- src/libstore/build/local-derivation-goal.cc | 2 ++ src/nix-build/nix-build.cc | 13 +++++++------ tests/structured-attrs.nix | 2 +- tests/structured-attrs.sh | 4 +--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9bb6f276c..e6b552b94 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1093,8 +1093,10 @@ void LocalDerivationGoal::writeStructuredAttrs() writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); + env["ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); + env["ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; } } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f91a56d6a..9bd7869f9 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -428,7 +428,6 @@ static void main_nix_build(int argc, char * * argv) env[var.first] = var.second; std::string structuredAttrsRC; - std::string exitCmd; if (env.count("__json")) { StorePathSet inputs; @@ -448,11 +447,13 @@ static void main_nix_build(int argc, char * * argv) if (auto structAttrs = parsedDrv.generateStructuredAttrs(std::nullopt, *store, inputs)) { auto val = structAttrs.value(); structuredAttrsRC = val.first; - auto attrsJSON = std::filesystem::current_path().string() + "/.attrs.json"; + auto attrsJSON = (Path) tmpDir + "/.attrs.json"; writeFile(attrsJSON, val.second.dump()); - exitCmd = "\n_rm_attrs_json() { rm -f " + attrsJSON + "; }" - + "\nexitHooks+=(_rm_attrs_json)" - + "\nfailureHooks+=(_rm_attrs_json)\n"; + auto attrsSH = (Path) tmpDir + "/.attrs.sh"; + writeFile(attrsSH, val.first); + env["ATTRS_SH_FILE"] = attrsSH; + env["ATTRS_JSON_FILE"] = attrsJSON; + keepTmp = true; } } @@ -471,7 +472,7 @@ static void main_nix_build(int argc, char * * argv) (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" "dontAddDisableDepTrack=1;\n" - + structuredAttrsRC + exitCmd + + + structuredAttrsRC + "\n[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "PATH=%4%:\"$PATH\"; " diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix index c39c3a346..f69ee45e9 100644 --- a/tests/structured-attrs.nix +++ b/tests/structured-attrs.nix @@ -36,7 +36,7 @@ mkDerivation { echo bar > $dest echo foo > $dest2 - json=$(cat .attrs.json) + json=$(cat $ATTRS_JSON_FILE) [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] [[ $json =~ '"narSize":288' ]] [[ $json =~ '"closureSize":288' ]] diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index c0fcb021a..241835539 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -10,7 +10,5 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result [[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] export NIX_BUILD_SHELL=$SHELL -[[ ! -e '.attrs.json' ]] env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < .attrs.json)"' -[[ ! -e '.attrs.json' ]] + --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $ATTRS_JSON_FILE)"' From 447928bdb55d160617387e7ac5954f9a8f36004b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 13 May 2021 01:20:49 +0200 Subject: [PATCH 088/555] Fix usage of structured attrs for `nix develop` --- src/nix/develop.cc | 5 +++++ src/nix/get-env.sh | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e51de2de8..3443fab73 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -256,6 +256,11 @@ struct Common : InstallableCommand, MixProfile // FIXME: properly unquote 'outputs'. StringMap rewrites; for (auto & outputName : tokenizeString>(replaceStrings(outputs->second.quoted, "'", ""))) { + std::regex ptrn(R"re(\[([A-z0-9]+)\]=.*)re"); + std::smatch match; + if (std::regex_match(outputName, match, ptrn)) { + outputName = match[1]; + } auto from = buildEnvironment.env.find(outputName); assert(from != buildEnvironment.env.end()); // FIXME: unquote diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 091c0f573..b6b8310a9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -8,7 +8,13 @@ if [[ -n $stdenv ]]; then source $stdenv/setup fi -for __output in $outputs; do +if [ -e .attrs.sh ]; then + __olist="${!outputs[@]}" +else + __olist=$outputs +fi + +for __output in $__olist; do if [[ -z $__done ]]; then export > ${!__output} set >> ${!__output} From f1e281c4fe1263a0c848bc8aaf57a0e61a99fa93 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 13 May 2021 16:11:56 +0200 Subject: [PATCH 089/555] Split shell & json creation for build environments with structured attrs --- src/libstore/build/local-derivation-goal.cc | 7 +++---- src/libstore/parsed-derivations.cc | 10 +++++++--- src/libstore/parsed-derivations.hh | 3 ++- src/nix-build/nix-build.cc | 13 ++++++++----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e6b552b94..60495ae3e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1086,10 +1086,9 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrs = parsedDrv->generateStructuredAttrs(inputRewrites, worker.store, inputPaths)) { - auto value = structAttrs.value(); - auto jsonSh = value.first; - auto json = value.second; + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(inputRewrites, worker.store, inputPaths)) { + auto json = structAttrsJson.value(); + auto jsonSh = parsedDrv->writeStructuredAttrsShell(json); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 029da8bd3..5675600c4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -124,8 +124,7 @@ bool ParsedDerivation::substitutesAllowed() const } static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); -std::optional ParsedDerivation::generateStructuredAttrs( - std::optional inputRewrites, Store & store, const StorePathSet & inputPaths) +std::optional ParsedDerivation::prepareStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); if (!structuredAttrs) return std::nullopt; @@ -163,6 +162,11 @@ std::optional ParsedDerivation::generateStructuredAt } } + return json; +} + +std::string ParsedDerivation::writeStructuredAttrsShell(nlohmann::json & json) +{ /* As a convenience to bash scripts, write a shell file that maps all attributes that are representable in bash - namely, strings, integers, nulls, Booleans, and arrays and @@ -229,6 +233,6 @@ std::optional ParsedDerivation::generateStructuredAt } } - return std::make_pair(jsonSh, json); + return jsonSh; } } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 4b8b8c8ff..37af36630 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -39,7 +39,8 @@ public: bool substitutesAllowed() const; - std::optional generateStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths); + std::optional prepareStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths); + std::string writeStructuredAttrsShell(nlohmann::json & json); }; } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9bd7869f9..4e4f27458 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -444,13 +444,16 @@ static void main_nix_build(int argc, char * * argv) drv ); - if (auto structAttrs = parsedDrv.generateStructuredAttrs(std::nullopt, *store, inputs)) { - auto val = structAttrs.value(); - structuredAttrsRC = val.first; + if (auto structAttrs = parsedDrv.prepareStructuredAttrs(std::nullopt, *store, inputs)) { + auto json = structAttrs.value(); + structuredAttrsRC = parsedDrv.writeStructuredAttrsShell(json); + auto attrsJSON = (Path) tmpDir + "/.attrs.json"; - writeFile(attrsJSON, val.second.dump()); + writeFile(attrsJSON, json.dump()); + auto attrsSH = (Path) tmpDir + "/.attrs.sh"; - writeFile(attrsSH, val.first); + writeFile(attrsSH, structuredAttrsRC); + env["ATTRS_SH_FILE"] = attrsSH; env["ATTRS_JSON_FILE"] = attrsJSON; keepTmp = true; From 3504c811a55ecd58e0712cf24829c67c192f5e80 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 18 May 2021 15:07:30 +0200 Subject: [PATCH 090/555] Add testcase for `nix develop` with `__structuredAttrs` --- src/nix/develop.cc | 3 +++ src/nix/get-env.sh | 9 ++++++--- tests/shell.nix | 8 ++++++++ tests/structured-attrs-shell.nix | 2 ++ tests/structured-attrs.sh | 5 +++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3443fab73..ee782c4ec 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -256,6 +256,9 @@ struct Common : InstallableCommand, MixProfile // FIXME: properly unquote 'outputs'. StringMap rewrites; for (auto & outputName : tokenizeString>(replaceStrings(outputs->second.quoted, "'", ""))) { + // Hacky way to obtain the key of an associate array. This is needed for strctured attrs where + // `outputs` is an associative array. If the regex isn't matched, the non-structured-attrs behavior will + // be used. std::regex ptrn(R"re(\[([A-z0-9]+)\]=.*)re"); std::smatch match; if (std::regex_match(outputName, match, ptrn)) { diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index b6b8310a9..431440516 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -8,6 +8,9 @@ if [[ -n $stdenv ]]; then source $stdenv/setup fi +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` +# must contain the array's keys (hence `${!...[@]}`) in this case. if [ -e .attrs.sh ]; then __olist="${!outputs[@]}" else @@ -16,10 +19,10 @@ fi for __output in $__olist; do if [[ -z $__done ]]; then - export > ${!__output} - set >> ${!__output} + export > "${!__output}" + set >> "${!__output}" __done=1 else - echo -n >> ${!__output} + echo -n >> "${!__output}" fi done diff --git a/tests/shell.nix b/tests/shell.nix index 24ebcc04c..53a32059b 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -8,6 +8,14 @@ let pkgs = rec { for pkg in $buildInputs; do export PATH=$PATH:$pkg/bin done + + # mimic behavior of stdenv for `$out` etc. for structured attrs. + if [ -n "''${ATTRS_SH_FILE}" ]; then + for o in "''${!outputs[@]}"; do + eval "''${o}=''${outputs[$o]}" + export "''${o}" + done + fi ''; stdenv = mkDerivation { diff --git a/tests/structured-attrs-shell.nix b/tests/structured-attrs-shell.nix index 0c01c2568..57c1e6bd2 100644 --- a/tests/structured-attrs-shell.nix +++ b/tests/structured-attrs-shell.nix @@ -6,10 +6,12 @@ let mkdir $out; echo bla > $out/bla ''; }; + inherit (import ./shell.nix { inNixShell = true; }) stdenv; in mkDerivation { name = "structured2"; __structuredAttrs = true; + inherit stdenv; outputs = [ "out" "dev" ]; my.list = [ "a" "b" "c" ]; exportReferencesGraph.refs = [ dep ]; diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 241835539..f851b3cbb 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -12,3 +12,8 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $ATTRS_JSON_FILE)"' + +# `nix develop` is a slightly special way of dealing with environment vars, it parses +# these from a shell-file exported from a derivation. This is to test especially `outputs` +# (which is an associative array in thsi case) being fine. +nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' From a92245b11026c884e76a1903abc0d47f84d79f5c Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 18 May 2021 15:17:02 +0200 Subject: [PATCH 091/555] Remove now-obsolete typedef --- src/libstore/parsed-derivations.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 37af36630..f02d5868e 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -6,8 +6,6 @@ namespace nix { -typedef std::pair StructuredAttrsWithShellRC; - class ParsedDerivation { StorePath drvPath; From 27ce722638eeabb987bc9b4a1234c2818c5bf401 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 22 Jun 2021 19:45:08 +0200 Subject: [PATCH 092/555] Prefix env vars for attrs.* files with NIX_ --- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/nix-build/nix-build.cc | 4 ++-- tests/shell.nix | 2 +- tests/structured-attrs.nix | 2 +- tests/structured-attrs.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 60495ae3e..de64737f4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1092,10 +1092,10 @@ void LocalDerivationGoal::writeStructuredAttrs() writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); - env["ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); - env["ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; + env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; } } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 4e4f27458..f67776083 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -454,8 +454,8 @@ static void main_nix_build(int argc, char * * argv) auto attrsSH = (Path) tmpDir + "/.attrs.sh"; writeFile(attrsSH, structuredAttrsRC); - env["ATTRS_SH_FILE"] = attrsSH; - env["ATTRS_JSON_FILE"] = attrsJSON; + env["NIX_ATTRS_SH_FILE"] = attrsSH; + env["NIX_ATTRS_JSON_FILE"] = attrsJSON; keepTmp = true; } } diff --git a/tests/shell.nix b/tests/shell.nix index 53a32059b..4581fa40a 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -10,7 +10,7 @@ let pkgs = rec { done # mimic behavior of stdenv for `$out` etc. for structured attrs. - if [ -n "''${ATTRS_SH_FILE}" ]; then + if [ -n "''${NIX_ATTRS_SH_FILE}" ]; then for o in "''${!outputs[@]}"; do eval "''${o}=''${outputs[$o]}" export "''${o}" diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix index f69ee45e9..e93139a44 100644 --- a/tests/structured-attrs.nix +++ b/tests/structured-attrs.nix @@ -36,7 +36,7 @@ mkDerivation { echo bar > $dest echo foo > $dest2 - json=$(cat $ATTRS_JSON_FILE) + json=$(cat $NIX_ATTRS_JSON_FILE) [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] [[ $json =~ '"narSize":288' ]] [[ $json =~ '"closureSize":288' ]] diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index f851b3cbb..9612020b8 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -11,7 +11,7 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $ATTRS_JSON_FILE)"' + --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` From 6f206549ba02c6f9bdbf9707bba9193a1d82e822 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 22 Jun 2021 20:37:25 +0200 Subject: [PATCH 093/555] Move `writeStructuredAttrsShell` out of `ParsedDerivation` class --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/parsed-derivations.cc | 12 ++++++------ src/libstore/parsed-derivations.hh | 3 ++- src/libstore/store-api.hh | 3 +++ src/nix-build/nix-build.cc | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index de64737f4..15a7cb4ac 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1088,7 +1088,7 @@ void LocalDerivationGoal::writeStructuredAttrs() { if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(inputRewrites, worker.store, inputPaths)) { auto json = structAttrsJson.value(); - auto jsonSh = parsedDrv->writeStructuredAttrsShell(json); + auto jsonSh = writeStructuredAttrsShell(json); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 5675600c4..f021e4de3 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -165,13 +165,13 @@ std::optional ParsedDerivation::prepareStructuredAttrs(std::opti return json; } -std::string ParsedDerivation::writeStructuredAttrsShell(nlohmann::json & json) +/* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ +std::string writeStructuredAttrsShell(nlohmann::json & json) { - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index f02d5868e..8061e3a8b 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -38,7 +38,8 @@ public: bool substitutesAllowed() const; std::optional prepareStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths); - std::string writeStructuredAttrsShell(nlohmann::json & json); }; +std::string writeStructuredAttrsShell(nlohmann::json & json); + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b742fed6d..7415cb54c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -695,6 +695,9 @@ public: const Stats & getStats(); + /* Computes the full closure of of a set of store-paths for e.g. + derivations that need this information for `exportReferencesGraph`. + */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); /* Return the build log of the specified store path, if available, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f67776083..459876465 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -446,7 +446,7 @@ static void main_nix_build(int argc, char * * argv) if (auto structAttrs = parsedDrv.prepareStructuredAttrs(std::nullopt, *store, inputs)) { auto json = structAttrs.value(); - structuredAttrsRC = parsedDrv.writeStructuredAttrsShell(json); + structuredAttrsRC = writeStructuredAttrsShell(json); auto attrsJSON = (Path) tmpDir + "/.attrs.json"; writeFile(attrsJSON, json.dump()); From c13d7d0b9770741cc7093dc71ec8fc7978171f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Jun 2021 16:37:45 +0200 Subject: [PATCH 094/555] Pass more values by reference Rather than copying them around everywhere Co-authored-by: Eelco Dolstra --- src/libstore/misc.cc | 4 ++-- src/libstore/realisation.cc | 4 ++-- src/libstore/realisation.hh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 80ee15c49..96d73b70e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -255,8 +255,8 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } std::map drvOutputReferences( - const std::set inputRealisations, - const StorePathSet pathReferences) + const std::set & inputRealisations, + const StorePathSet & pathReferences) { std::map res; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index d2d306476..0d9d4b433 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -22,14 +22,14 @@ std::string DrvOutput::to_string() const { return strHash() + "!" + outputName; } -std::set Realisation::closure(Store & store, std::set startOutputs) +std::set Realisation::closure(Store & store, const std::set & startOutputs) { std::set res; Realisation::closure(store, startOutputs, res); return res; } -void Realisation::closure(Store & store, std::set startOutputs, std::set & res) +void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) { auto getDeps = [&](const Realisation& current) -> std::set { std::set res; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 8cda5a752..7fdb65acd 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -44,8 +44,8 @@ struct Realisation { bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; - static std::set closure(Store &, std::set); - static void closure(Store &, std::set, std::set& res); + static std::set closure(Store &, const std::set &); + static void closure(Store &, const std::set &, std::set& res); StorePath getPath() const { return outPath; } From 8d09a4f9a09473e6a32cc118ad10827ec5650700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 22 Jun 2021 09:31:25 +0200 Subject: [PATCH 095/555] Remove a useless string split Co-authored-by: Eelco Dolstra --- src/libstore/store-api.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6f29c430a..7e057b1ea 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -803,8 +803,7 @@ std::map copyPaths(ref srcStore, ref dstStor if (!currentChild) throw Error( "Incomplete realisation closure: '%s' is a " - "dependency " - "of '%s' but isn’t registered", + "dependency of '%s' but isn’t registered", drvOutput.to_string(), current.id.to_string()); children.insert(*currentChild); } From 7c96a76dd7da69fa527dea7110d1156c1c5fbe9e Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Jun 2021 09:26:55 +0200 Subject: [PATCH 096/555] Reformat the sql statements --- src/libstore/local-store.cc | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 45bb5efc8..b56df735c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -713,14 +713,18 @@ void LocalStore::registerDrvOutput(const Realisation & info) settings.requireExperimentalFeature("ca-derivations"); retrySQLite([&]() { auto state(_state.lock()); - state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath( - info.outPath))(concatStringsSep(" ", info.signatures)) + state->stmts->RegisterRealisedOutput.use() + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) .exec(); uint64_t myId = state->db.getLastInsertedRowId(); for (auto & [outputId, _] : info.dependentRealisations) { - state->stmts->AddRealisationReference - .use()(myId)(outputId.strHash())(outputId.outputName) + state->stmts->AddRealisationReference.use() + (myId) + (outputId.strHash()) + (outputId.outputName) .exec(); } }); @@ -1721,8 +1725,9 @@ std::optional LocalStore::queryRealisation( typedef std::optional Ret; return retrySQLite([&]() -> Ret { auto state(_state.lock()); - auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()( - id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use() + (id.strHash()) + (id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); @@ -1732,13 +1737,14 @@ std::optional LocalStore::queryRealisation( std::map dependentRealisations; auto useRealisationRefs( - state->stmts->QueryRealisationReferences.use()( - realisationDbId)); + state->stmts->QueryRealisationReferences.use() + (realisationDbId)); while (useRealisationRefs.next()) { auto depHash = useRealisationRefs.getStr(0); auto depOutputName = useRealisationRefs.getStr(1); - auto useQueryRealisedOutput( - state->stmts->QueryRealisedOutput.use()(depHash)(depOutputName)); + auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use() + (depHash) + (depOutputName)); assert(useQueryRealisedOutput.next()); auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName }; From ed0e21a88d64d772304d079c39d76feca41c02d3 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Jun 2021 07:45:41 +0200 Subject: [PATCH 097/555] Fix indentation --- src/libstore/local-store.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b56df735c..348d964ca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -714,10 +714,10 @@ void LocalStore::registerDrvOutput(const Realisation & info) retrySQLite([&]() { auto state(_state.lock()); state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) .exec(); uint64_t myId = state->db.getLastInsertedRowId(); for (auto & [outputId, _] : info.dependentRealisations) { From a5df669bc685834b16de0ab57723ff734c10d2f7 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 13:35:46 +0200 Subject: [PATCH 098/555] =?UTF-8?q?Add=20a=20test=20for=20the=20=E2=80=9Ct?= =?UTF-8?q?wo=20glibc=E2=80=9D=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/ca/duplicate-realisation-in-closure.sh | 26 +++++++++++++++ tests/ca/nondeterministic.nix | 35 ++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 62 insertions(+) create mode 100644 tests/ca/duplicate-realisation-in-closure.sh create mode 100644 tests/ca/nondeterministic.nix diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh new file mode 100644 index 000000000..bfe2a4e08 --- /dev/null +++ b/tests/ca/duplicate-realisation-in-closure.sh @@ -0,0 +1,26 @@ +source ./common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +rm -rf $REMOTE_STORE_DIR +clearStore + +# Build dep1 and push that to the binary cache. +# This entails building (and pushing) current-time. +nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1 +clearStore +sleep 2 # To make sure that `$(date)` will be different +# Build dep2. +# As we’ve cleared the cache, we’ll have to rebuild current-time. And because +# the current time isn’t the same as before, this will yield a new (different) +# realisation +nix build -f nondeterministic.nix dep2 + +# Build something that depends both on dep1 and dep2. +# If everything goes right, we should rebuild dep2 rather than fetch it from +# the cache (because that would mean duplicating `current-time` in the closure), +# and have `dep1 == dep2`. +nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs diff --git a/tests/ca/nondeterministic.nix b/tests/ca/nondeterministic.nix new file mode 100644 index 000000000..d6d099a3e --- /dev/null +++ b/tests/ca/nondeterministic.nix @@ -0,0 +1,35 @@ +with import ./config.nix; + +let mkCADerivation = args: mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; +} // args); +in + +rec { + currentTime = mkCADerivation { + name = "current-time"; + buildCommand = '' + mkdir $out + echo $(date) > $out/current-time + ''; + }; + dep = seed: mkCADerivation { + name = "dep"; + inherit seed; + buildCommand = '' + echo ${currentTime} > $out + ''; + }; + dep1 = dep 1; + dep2 = dep 2; + toplevel = mkCADerivation { + name = "toplevel"; + buildCommand = '' + test ${dep1} == ${dep2} + touch $out + ''; + }; +} + diff --git a/tests/local.mk b/tests/local.mk index 0d594cda0..e7161d298 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -47,6 +47,7 @@ nix_tests = \ compute-levels.sh \ ca/build.sh \ ca/build-with-garbage-path.sh \ + ca/duplicate-realisation-in-closure.sh \ ca/substitute.sh \ ca/signatures.sh \ ca/nix-run.sh \ From b8f7177a7b2e884cbfb8bbe3c1ee8d586159fbb3 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 16:19:46 +0200 Subject: [PATCH 099/555] Properly fail when trying to register an incoherent realisation --- src/libstore/local-store.cc | 136 +++++++++++++++++++++++++----------- src/libstore/local-store.hh | 2 + src/libstore/realisation.cc | 6 ++ src/libstore/realisation.hh | 2 + 4 files changed, 107 insertions(+), 39 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c2256635a..8df1d55b9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -53,6 +53,7 @@ struct LocalStore::State::Stmts { SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; + SQLiteStmt UpdateRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryRealisedOutput; @@ -345,6 +346,15 @@ LocalStore::LocalStore(const Params & params) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); + state->stmts->UpdateRealisedOutput.create(state->db, + R"( + update Realisations + set signatures = ? + where + drvPath = ? and + outputName = ? + ; + )"); state->stmts->QueryRealisedOutput.create(state->db, R"( select Realisations.id, Output.path, Realisations.signatures from Realisations @@ -710,14 +720,41 @@ void LocalStore::registerDrvOutput(const Realisation & info) settings.requireExperimentalFeature("ca-derivations"); retrySQLite([&]() { auto state(_state.lock()); - state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) - .exec(); + if (auto oldR = queryRealisation_(*state, info.id)) { + if (info.isCompatibleWith(*oldR)) { + auto combinedSignatures = oldR->signatures; + combinedSignatures.insert(info.signatures.begin(), + info.signatures.end()); + state->stmts->UpdateRealisedOutput.use() + (concatStringsSep(" ", combinedSignatures)) + (info.id.strHash()) + (info.id.outputName) + .exec(); + } else { + throw Error("Trying to register a realisation of '%s', but we already " + "have another one locally", + info.id.to_string()); + } + } else { + state->stmts->RegisterRealisedOutput.use() + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) + .exec(); + } uint64_t myId = state->db.getLastInsertedRowId(); - for (auto & [outputId, _] : info.dependentRealisations) { + for (auto & [outputId, depPath] : info.dependentRealisations) { + auto localRealisation = queryRealisationCore_(*state, outputId); + if (!localRealisation) + throw Error("unable to register the derivation '%s' as it " + "depends on the non existent '%s'", + info.id.to_string(), outputId.to_string()); + if (localRealisation->second.outPath != depPath) + throw Error("unable to register the derivation '%s' as it " + "depends on a realisation of '%s' that doesn’t" + "match what we have locally", + info.id.to_string(), outputId.to_string()); state->stmts->AddRealisationReference.use() (myId) (outputId.strHash()) @@ -1734,46 +1771,67 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryRealisation( - const DrvOutput& id) { - typedef std::optional Ret; - return retrySQLite([&]() -> Ret { - auto state(_state.lock()); - auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use() +std::optional> LocalStore::queryRealisationCore_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto useQueryRealisedOutput( + state.stmts->QueryRealisedOutput.use() (id.strHash()) (id.outputName)); - if (!useQueryRealisedOutput.next()) - return std::nullopt; - auto realisationDbId = useQueryRealisedOutput.getInt(0); - auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); - auto signatures = - tokenizeString(useQueryRealisedOutput.getStr(2)); + if (!useQueryRealisedOutput.next()) + return std::nullopt; + auto realisationDbId = useQueryRealisedOutput.getInt(0); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto signatures = + tokenizeString(useQueryRealisedOutput.getStr(2)); - std::map dependentRealisations; - auto useRealisationRefs( - state->stmts->QueryRealisationReferences.use() - (realisationDbId)); - while (useRealisationRefs.next()) { - auto depHash = useRealisationRefs.getStr(0); - auto depOutputName = useRealisationRefs.getStr(1); - auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use() - (depHash) - (depOutputName)); - assert(useQueryRealisedOutput.next()); - auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); - auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName }; - dependentRealisations.insert({depId, outputPath}); - } - - return Ret{Realisation{ + return {{ + realisationDbId, + Realisation{ .id = id, .outPath = outputPath, .signatures = signatures, - .dependentRealisations = dependentRealisations, - }}; - }); + } + }}; } +std::optional LocalStore::queryRealisation_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto maybeCore = queryRealisationCore_(state, id); + if (!maybeCore) + return std::nullopt; + auto [realisationDbId, res] = *maybeCore; + + std::map dependentRealisations; + auto useRealisationRefs( + state.stmts->QueryRealisationReferences.use() + (realisationDbId)); + while (useRealisationRefs.next()) { + auto depId = DrvOutput { + Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), + useRealisationRefs.getStr(1), + }; + auto dependentRealisation = queryRealisationCore_(state, depId); + assert(dependentRealisation); // Enforced by the db schema + auto outputPath = dependentRealisation->second.outPath; + dependentRealisations.insert({depId, outputPath}); + } + + res.dependentRealisations = dependentRealisations; + + return { res }; +} + +std::optional +LocalStore::queryRealisation(const DrvOutput &id) { + return retrySQLite>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); +} FixedOutputHash LocalStore::hashCAPath( const FileIngestionMethod & method, const HashType & hashType, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 15c7fc306..a01d48c4b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,8 @@ public: void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); + std::optional queryRealisation_(State & state, const DrvOutput & id); + std::optional> queryRealisationCore_(State & state, const DrvOutput & id); std::optional queryRealisation(const DrvOutput&) override; private: diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 0d9d4b433..76aec74ce 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -140,6 +140,12 @@ StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } +bool Realisation::isCompatibleWith(const Realisation & other) const +{ + assert (id == other.id); + return outPath == other.outPath; +} + void RealisedPath::closure( Store& store, const RealisedPath::Set& startPaths, diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 7fdb65acd..05d2bc44f 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -47,6 +47,8 @@ struct Realisation { static std::set closure(Store &, const std::set &); static void closure(Store &, const std::set &, std::set& res); + bool isCompatibleWith(const Realisation & other) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); From d32cf0c17a3e5e139c384375084d9eb6aa428c3b Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 19 May 2021 16:27:09 +0200 Subject: [PATCH 100/555] Gracefully ignore a substituter if it holds an incompatible realisation --- .../build/drv-output-substitution-goal.cc | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 1703e845d..ec3a8d758 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker void DrvOutputSubstitutionGoal::init() { trace("init"); + + /* If the derivation already exists, we’re done */ + if (worker.store.queryRealisation(id)) { + amDone(ecSuccess); + return; + } + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); tryNext(); } @@ -53,9 +60,18 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - for (const auto & [drvOutputDep, _] : outputInfo->dependentRealisations) { - if (drvOutputDep != id) { - addWaitee(worker.makeDrvOutputSubstitutionGoal(drvOutputDep)); + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring", + sub->getUri(), + depId.to_string()); + tryNext(); + return; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); } } From 40f925b2dacb481b62d325fb41641804524a5dc8 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Jun 2021 10:42:23 +0200 Subject: [PATCH 101/555] Fix indentation --- src/libstore/local-store.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8df1d55b9..064f7a432 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1826,11 +1826,12 @@ std::optional LocalStore::queryRealisation_( } std::optional -LocalStore::queryRealisation(const DrvOutput &id) { - return retrySQLite>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); +LocalStore::queryRealisation(const DrvOutput & id) +{ + return retrySQLite>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); } FixedOutputHash LocalStore::hashCAPath( From 16fb7d8d95a8bc81e7df885ab4167c8a03f1dddf Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Jun 2021 10:46:29 +0200 Subject: [PATCH 102/555] Display the diverging paths in case of a realisation mismatch --- src/libstore/build/drv-output-substitution-goal.cc | 9 +++++++-- src/libstore/local-store.cc | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index ec3a8d758..be270d079 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -65,9 +65,14 @@ void DrvOutputSubstitutionGoal::tryNext() if (auto localOutputInfo = worker.store.queryRealisation(depId); localOutputInfo && localOutputInfo->outPath != depPath) { warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring", + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", sub->getUri(), - depId.to_string()); + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); tryNext(); return; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 064f7a432..d7c7f8e1d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -732,8 +732,13 @@ void LocalStore::registerDrvOutput(const Realisation & info) .exec(); } else { throw Error("Trying to register a realisation of '%s', but we already " - "have another one locally", - info.id.to_string()); + "have another one locally.\n" + "Local: %s\n" + "Remote: %s", + info.id.to_string(), + printStorePath(oldR->outPath), + printStorePath(info.outPath) + ); } } else { state->stmts->RegisterRealisedOutput.use() From c878cee8954151aaa1054af7ef3746a979b05832 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Jun 2021 10:50:28 +0200 Subject: [PATCH 103/555] Assert that compatible realisations have the same dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Should always hold, but that’s not necessarily obvious, so better enforce it --- src/libstore/realisation.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 76aec74ce..eadec594c 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -143,7 +143,11 @@ StorePath RealisedPath::path() const { bool Realisation::isCompatibleWith(const Realisation & other) const { assert (id == other.id); - return outPath == other.outPath; + if (outPath == other.outPath) { + assert(dependentRealisations == other.dependentRealisations); + return true; + } + return false; } void RealisedPath::closure( From 4f80464645e4c8e7ca9455fc53cc76dc50f688ed Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 1 Jun 2021 07:58:21 +0000 Subject: [PATCH 104/555] Apply OS checks to host platform, not build Previously, the build system used uname(1) output when it wanted to check the operating system it was being built for, which meant that it didn't take into-account cross-compilation when the build and host operating systems were different. To fix this, instead of consulting uname output, we consult the host triple, specifically the third "kernel" part. For "kernel"s with stable ABIs, like Linux or Cygwin, we can use a simple ifeq to test whether we're compiling for that system, but for other platforms, like Darwin, FreeBSD, or Solaris, we have to use a more complicated check to take into account the version numbers at the end of the "kernel"s. I couldn't find a way to just strip these version numbers in GNU Make without shelling out, which would be even more ugly IMO. Because these checks differ between kernels, and the patsubst ones are quite fiddly, I've added variables for each host OS we might want to check to make them easier to reuse. --- Makefile.config.in | 1 + misc/launchd/local.mk | 2 +- misc/systemd/local.mk | 2 +- misc/upstart/local.mk | 2 +- mk/lib.mk | 27 +++++++++++++++++++----- mk/libraries.mk | 18 ++++++++-------- nix-rust/local.mk | 8 +++---- perl/Makefile.config.in | 1 + perl/configure.ac | 2 ++ perl/local.mk | 2 +- src/libexpr/local.mk | 2 +- src/libstore/local.mk | 6 +++--- src/resolve-system-dependencies/local.mk | 2 +- 13 files changed, 48 insertions(+), 27 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index fd92365eb..c8c4446b4 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ BOOST_LDFLAGS = @BOOST_LDFLAGS@ diff --git a/misc/launchd/local.mk b/misc/launchd/local.mk index 0ba722efb..a39188fe6 100644 --- a/misc/launchd/local.mk +++ b/misc/launchd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons)) diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 785db52a4..1fa037485 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index 5071676dc..2fbfb29b9 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) diff --git a/mk/lib.mk b/mk/lib.mk index 975102531..92f0983d5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,8 +10,25 @@ bin-scripts := noinst-scripts := man-pages := install-tests := -OS = $(shell uname -s) +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + endif +endif # Hack to define a literal space. space := @@ -50,16 +67,16 @@ endif BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE else GLOBAL_CFLAGS += -fPIC GLOBAL_CXXFLAGS += -fPIC endif - ifneq ($(OS), Darwin) - ifneq ($(OS), SunOS) - ifneq ($(OS), FreeBSD) + ifndef HOST_DARWIN + ifndef HOST_SOLARIS + ifndef HOST_FREEBSD GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries endif endif diff --git a/mk/libraries.mk b/mk/libraries.mk index 7c0e4f100..07bd54dab 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -1,9 +1,9 @@ libs-list := -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN SO_EXT = dylib else - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN SO_EXT = dll else SO_EXT = so @@ -59,7 +59,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -73,18 +73,18 @@ define build-library ifeq ($(BUILD_SHARED_LIBS), 1) ifdef $(1)_ALLOW_UNDEFINED - ifeq ($(OS), Darwin) + ifdef HOST_DARWIN $(1)_LDFLAGS += -undefined suppress -flat_namespace endif else - ifneq ($(OS), Darwin) - ifneq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifndef HOST_DARWIN + ifndef HOST_CYGWIN $(1)_LDFLAGS += -Wl,-z,defs endif endif endif - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT) endif @@ -93,7 +93,7 @@ define build-library $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) endif $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) @@ -108,7 +108,7 @@ define build-library $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN ifeq ($(SET_RPATH_TO_LIBS), 1) $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) else diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 9650cdf93..538244594 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -11,12 +11,12 @@ libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ifeq ($(OS), Linux) +ifdef HOST_LINUX libnixrust_LDFLAGS_USE += -ldl libnixrust_LDFLAGS_USE_INSTALLED += -ldl endif -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" else libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) @@ -31,7 +31,7 @@ $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml $(libnixrust_INSTALL_PATH): $(libnixrust_PATH) $(target-gen) cp $^ $@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN install_name_tool -id $@ $@ endif @@ -40,7 +40,7 @@ clean: clean-rust clean-rust: $(suppress) rm -rfv nix-rust/target -ifneq ($(OS), Darwin) +ifndef HOST_DARWIN check: rust-tests rust-tests: diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index eccfbd9f6..d856de3ad 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ diff --git a/perl/configure.ac b/perl/configure.ac index 85183c005..eb65ac17b 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -7,6 +7,8 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX +AC_CANONICAL_HOST + # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE diff --git a/perl/local.mk b/perl/local.mk index b13d4c0d6..0eae651d8 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -28,7 +28,7 @@ Store_CXXFLAGS = \ Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) -ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) +ifdef HOST_CYGWIN archlib = $(shell perl -E 'use Config; print $$Config{archlib};') libperl = $(shell perl -E 'use Config; print $$Config{libperl};') Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index c40abfb78..1aed8e152 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = -lboost_context -ifeq ($(OS), Linux) +ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b6652984c..2fc334a82 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,11 +9,11 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifeq ($(OS), Linux) +ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb endif @@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1) libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif -ifeq ($(OS), SunOS) +ifdef HOST_SOLARIS libstore_LDFLAGS += -lsocket endif diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 054ae01cb..fc48a8417 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN programs += resolve-system-dependencies endif From fd3f5e9085737734d8769c21559461c26dc5a165 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Jun 2021 17:36:50 +0200 Subject: [PATCH 105/555] Make the CA tests actuall test CA derivations Fix a mistake in config.nix that was preventing `NIX_TESTS_CA_BY_DEFAULT` from having any meaningful effect --- tests/config.nix.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config.nix.in b/tests/config.nix.in index 9b00d9ddb..7facbdcbc 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -22,6 +22,6 @@ rec { builder = shell; args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; - } // removeAttrs args ["builder" "meta"]) + } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; -} // caArgs +} From 01a3f4d7ec79d434dc34b64a761ddf681fe63a48 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Jun 2021 17:37:29 +0200 Subject: [PATCH 106/555] Fix the CA gc test Broken by https://github.com/NixOS/nix/issues/4936 --- tests/gc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gc.sh b/tests/gc.sh index 8b4f8d282..cf0e2c32d 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo nix-store --gc --print-roots | grep $outPath nix-store --gc --print-live | grep $outPath nix-store --gc --print-dead | grep $drvPath -if nix-store --gc --print-dead | grep $outPath; then false; fi +if nix-store --gc --print-dead | grep -E $outPath$; then false; fi nix-store --gc --print-dead From bee71e1bb1c63dc730e31523dd8075c922153051 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 2 Jun 2021 00:45:03 +0200 Subject: [PATCH 107/555] Add a fish completion script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is only rudimentary support as allowed by `NIX_GET_COMPLETIONS`. In the future, we could use complete’s `--wraps` argument to autocomplete arguments for programs after `nix shell -c`. --- Makefile | 1 + misc/fish/completion.fish | 37 +++++++++++++++++++++++++++++++++++++ misc/fish/local.mk | 1 + 3 files changed, 39 insertions(+) create mode 100644 misc/fish/completion.fish create mode 100644 misc/fish/local.mk diff --git a/Makefile b/Makefile index b7f0e79db..dd259e5cd 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ makefiles = \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ misc/bash/local.mk \ + misc/fish/local.mk \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish new file mode 100644 index 000000000..bedbefaf8 --- /dev/null +++ b/misc/fish/completion.fish @@ -0,0 +1,37 @@ +function _nix_complete + # Get the current command up to a cursor. + # - Behaves correctly even with pipes and nested in commands like env. + # - TODO: Returns the command verbatim (does not interpolate variables). + # That might not be optimal for arguments like -f. + set -l nix_args (commandline --current-process --tokenize --cut-at-cursor) + # --cut-at-cursor with --tokenize removes the current token so we need to add it separately. + # https://github.com/fish-shell/fish-shell/issues/7375 + # Can be an empty string. + set -l current_token (commandline --current-token --cut-at-cursor) + + # Nix wants the index of the argv item to complete but the $nix_args variable + # also contains the program name (argv[0]) so we would need to subtract 1. + # But the variable also misses the current token so it cancels out. + set -l nix_arg_to_complete (count $nix_args) + + env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token +end + +function _nix_accepts_files + set -l response (_nix_complete) + # First line is either filenames or no-filenames. + test $response[1] = 'filenames' +end + +function _nix + set -l response (_nix_complete) + # Skip the first line since it handled by _nix_accepts_files. + # Tail lines each contain a command followed by a tab character and, optionally, a description. + # This is also the format fish expects. + string collect -- $response[2..-1] +end + +# Disable file path completion if paths do not belong in the current context. +complete --command nix --condition 'not _nix_accepts_files' --no-files + +complete --command nix --arguments '(_nix)' diff --git a/misc/fish/local.mk b/misc/fish/local.mk new file mode 100644 index 000000000..ece899fc3 --- /dev/null +++ b/misc/fish/local.mk @@ -0,0 +1 @@ +$(eval $(call install-file-as, $(d)/completion.fish, $(datarootdir)/fish/vendor_completions.d/nix.fish, 0644)) From e3d11f9a9ca8cf1663de0182154683db250df095 Mon Sep 17 00:00:00 2001 From: Thomas Churchman Date: Wed, 23 Jun 2021 22:07:55 +0100 Subject: [PATCH 108/555] Improve machine store URI parsing --- src/libstore/machines.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index b42e5e434..9843ccf04 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : storeUri( - // Backwards compatibility: if the URI is a hostname, - // prepend ssh://. + // Backwards compatibility: if the URI is schemeless, is not a path, + // and is not one of the special store connection words, prepend + // ssh://. storeUri.find("://") != std::string::npos - || hasPrefix(storeUri, "local") - || hasPrefix(storeUri, "remote") - || hasPrefix(storeUri, "auto") - || hasPrefix(storeUri, "/") + || storeUri.find("/") != std::string::npos + || storeUri == "auto" + || storeUri == "daemon" + || storeUri == "local" + || hasPrefix(storeUri, "auto?") + || hasPrefix(storeUri, "daemon?") + || hasPrefix(storeUri, "local?") + || hasPrefix(storeUri, "?") ? storeUri : "ssh://" + storeUri), systemTypes(systemTypes), From be7a4a6a1303ddbd1f8942f02c04f69016c51da0 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 24 Jun 2021 11:41:51 +0200 Subject: [PATCH 109/555] Make the post-build-hook also run for unresolved CA derivations Fix #4837 --- src/libstore/build/derivation-goal.cc | 127 +++++++++++++++----------- tests/ca/post-hook.sh | 11 +++ tests/local.mk | 1 + tests/post-hook.sh | 2 +- 4 files changed, 89 insertions(+), 52 deletions(-) create mode 100755 tests/ca/post-hook.sh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8c9ef0101..73d1ed7cc 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -739,6 +739,63 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() { } +void runPostBuildHook( + Store & store, + Logger & logger, + const StorePath & drvPath, + StorePathSet outputPaths +) +{ + auto hook = settings.postBuildHook; + if (hook == "") + return; + + Activity act(logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{store.printStorePath(drvPath)}); + PushActivity pact(act.id); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (std::string_view data) override { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); +} void DerivationGoal::buildDone() { @@ -804,57 +861,15 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (std::string_view data) override { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } + StorePathSet outputPaths; + for (auto & [_, path] : finalOutputs) + outputPaths.insert(path); + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); if (buildMode == bmCheck) { cleanupPostOutputsRegisteredModeCheck(); @@ -910,6 +925,8 @@ void DerivationGoal::resolvedFinished() { auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + StorePathSet outputPaths; + // `wantedOutputs` might be empty, which means “all the outputs” auto realWantedOutputs = wantedOutputs; if (realWantedOutputs.empty()) @@ -930,6 +947,7 @@ void DerivationGoal::resolvedFinished() { newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); + outputPaths.insert(realisation->outPath); } else { // If we don't have a realisation, then it must mean that something // failed when building the resolved drv @@ -937,6 +955,13 @@ void DerivationGoal::resolvedFinished() { } } + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + // This is potentially a bit fishy in terms of error reporting. Not sure // how to do it in a cleaner way amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); diff --git a/tests/ca/post-hook.sh b/tests/ca/post-hook.sh new file mode 100755 index 000000000..4b8da4cd8 --- /dev/null +++ b/tests/ca/post-hook.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./post-hook.sh + + diff --git a/tests/local.mk b/tests/local.mk index 82cec1df3..35ee9e271 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -39,6 +39,7 @@ nix_tests = \ search.sh \ nix-copy-ssh.sh \ post-hook.sh \ + ca/post-hook.sh \ function-trace.sh \ recursive.sh \ describe-stores.sh \ diff --git a/tests/post-hook.sh b/tests/post-hook.sh index aa3e6a574..238a8f826 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -4,7 +4,7 @@ clearStore rm -f $TEST_ROOT/result -export REMOTE_STORE=$TEST_ROOT/remote_store +export REMOTE_STORE=file:$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh From bec83a6f953ab045e85d50b0cab9de176cd00bc0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 16 Nov 2020 11:03:53 +0100 Subject: [PATCH 110/555] BoehmGCStackAllocator: ignore stack protection page This fixes a crash that looks like: ``` Thread 1 "nix-build" received signal SIGSEGV, Segmentation fault. 0x00007ffff7ad22a0 in GC_push_all_eager () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 (gdb) bt 0 0x00007ffff7ad22a0 in GC_push_all_eager () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 1 0x00007ffff7adeefb in GC_push_all_stacks () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 2 0x00007ffff7ad5ac7 in GC_mark_some () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 3 0x00007ffff7ad77bd in GC_stopped_mark () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 4 0x00007ffff7adbe3a in GC_try_to_collect_inner.part.0 () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 5 0x00007ffff7adc2a2 in GC_collect_or_expand () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 6 0x00007ffff7adc4f8 in GC_allocobj () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 7 0x00007ffff7adc88f in GC_generic_malloc_inner () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 8 0x00007ffff7ae1a04 in GC_generic_malloc_many () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 9 0x00007ffff7ae1c72 in GC_malloc_kind () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1 10 0x00007ffff7e003d6 in nix::EvalState::allocValue() () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 11 0x00007ffff7e04b9c in nix::EvalState::callPrimOp(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 12 0x00007ffff7e0a773 in nix::EvalState::callFunction(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 13 0x00007ffff7e0a91d in nix::ExprApp::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 14 0x00007ffff7e0a8f8 in nix::ExprApp::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 15 0x00007ffff7e0e0e8 in nix::ExprOpNEq::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 16 0x00007ffff7e0d708 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 17 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 18 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 19 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 20 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 21 0x00007ffff7e09e19 in nix::ExprOpNot::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 22 0x00007ffff7e0a792 in nix::EvalState::callFunction(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 23 0x00007ffff7e8cba0 in nix::addPath(nix::EvalState&, nix::Pos const&, std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&, nix::Value*, nix::FileIngestionMethod, std::optional, nix::Value&)::{lambda(std::__cxx11::basic_string, std::allocator > const&)#1}::operator()(std::__cxx11::basic_string, std::allocator > const&) const () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so 24 0x00007ffff752e6f9 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 25 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 26 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 27 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 28 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 29 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string, std::allocator > const&, nix::Sink&, std::function, std::allocator > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 30 0x00007ffff757f8c0 in void boost::context::detail::fiber_entry, std::allocator > >::control_block::control_block, std::function)::SinkToSource::read(char*, unsigned long)::{lambda(boost::coroutines2::detail::push_coroutine, std::allocator > >&)#1}>(boost::context::preallocated, nix::VirtualStackAllocator&&, nix::sinkToSource(std::function, std::function)::SinkToSource::read(char*, unsigned long)::{lambda(boost::coroutines2::detail::push_coroutine, std::allocator > >&)#1}&&)::{lambda(boost::context::fiber&&)#1}> >(boost::context::detail::transfer_t) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so 31 0x00007ffff6f331ef in make_fcontext () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libboost_context.so.1.69.0 32 0x0000000000000000 in ?? () ``` --- src/libexpr/eval.cc | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ef9f8efca..c078bf4a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -233,22 +233,34 @@ static void * oomHandler(size_t requested) } class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) }; + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { + return sctx.size - boost::context::stack_traits::page_size(); + } + public: boost::context::stack_context allocate() override { auto sctx = stack.allocate(); - GC_add_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); return sctx; } void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); stack.deallocate(sctx); } From 7746cb13dc4e644c9792b3e3666cd49635d694e0 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Jun 2021 17:27:18 +0200 Subject: [PATCH 111/555] Make CA derivations compatible with recursive Nix Add an access-control list to the realisations in recursive-nix (similar to the already existing one for store paths), so that we can build content-addressed derivations in the restricted store. Fix #4353 --- src/libstore/build/local-derivation-goal.cc | 21 +++++++++++++++++++-- src/libstore/build/local-derivation-goal.hh | 8 ++++++++ tests/ca/recursive.sh | 11 +++++++++++ tests/local.mk | 1 + tests/recursive.sh | 17 +++++++++-------- 5 files changed, 48 insertions(+), 10 deletions(-) create mode 100755 tests/ca/recursive.sh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 279139020..ba0aca29c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1333,13 +1333,18 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo std::optional queryRealisation(const DrvOutput & id) override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation - { throw Error("queryRealisation"); } + { + if (!goal.isAllowed(id)) + throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); + return next->queryRealisation(id); + } void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; + std::set newRealisations; for (auto & req : paths) { if (!goal.isAllowed(req)) @@ -1352,16 +1357,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo auto p = std::get_if(&path); if (!p) continue; auto & bfd = *p; + auto drv = readDerivation(bfd.drvPath); + auto drvHashes = staticOutputHashes(*this, drv); auto outputs = next->queryDerivationOutputMap(bfd.drvPath); for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) + if (wantOutput(outputName, bfd.outputs)) { newPaths.insert(outputPath); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto thisRealisation = next->queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName} + ); + assert(thisRealisation); + newRealisations.insert(*thisRealisation); + } + } } StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index d30be2351..088a57209 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal /* Paths that were added via recursive Nix calls. */ StorePathSet addedPaths; + /* Realisations that were added via recursive Nix calls. */ + std::set addedDrvOutputs; + /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/' where @@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const DrvOutput & id) + { + return addedDrvOutputs.count(id); + } + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/tests/ca/recursive.sh b/tests/ca/recursive.sh new file mode 100755 index 000000000..d9281d91f --- /dev/null +++ b/tests/ca/recursive.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./recursive.sh + + diff --git a/tests/local.mk b/tests/local.mk index 82cec1df3..4d9d314cc 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -52,6 +52,7 @@ nix_tests = \ ca/signatures.sh \ ca/nix-shell.sh \ ca/nix-run.sh \ + ca/recursive.sh \ ca/nix-copy.sh # parallel.sh diff --git a/tests/recursive.sh b/tests/recursive.sh index a55b061b5..b6740877d 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -9,9 +9,9 @@ rm -f $TEST_ROOT/result export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' with import ./config.nix; - mkDerivation { + mkDerivation rec { name = "recursive"; dummy = builtins.toFile "dummy" "bla bla"; SHELL = shell; @@ -19,11 +19,13 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r # Note: this is a string without context. unreachable = builtins.getEnv "unreachable"; + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + requiredSystemFeatures = [ "recursive-nix" ]; buildCommand = '\'\'' mkdir $out - opts="--experimental-features nix-command" + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH @@ -46,16 +48,15 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r # Add it to our closure. ln -s $foobar $out/foobar - [[ $(nix $opts path-info --all | wc -l) -eq 3 ]] + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] # Build a derivation. nix $opts build -L --impure --expr '\'' - derivation { + with import ${./config.nix}; + mkDerivation { name = "inner1"; - builder = builtins.getEnv "SHELL"; - system = builtins.getEnv "system"; + buildCommand = "echo $fnord blaat > $out"; fnord = builtins.toFile "fnord" "fnord"; - args = [ "-c" "echo $fnord blaat > $out" ]; } '\'' From 644415d3912633773d2c8f219572fbfa452f4b56 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 24 Jun 2021 15:06:07 +0200 Subject: [PATCH 112/555] Perform input rewrites only in LocalDerivationGoal --- src/libstore/build/local-derivation-goal.cc | 11 ++++++++++- src/libstore/parsed-derivations.cc | 11 ++--------- src/libstore/parsed-derivations.hh | 2 +- src/nix-build/nix-build.cc | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 15a7cb4ac..259134eba 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1086,8 +1086,17 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(inputRewrites, worker.store, inputPaths)) { + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { auto json = structAttrsJson.value(); + nlohmann::json rewritten; + for (auto & [i, v] : json["outputs"].get()) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + rewritten[i] = rewriteStrings(v, inputRewrites); + } + + json["outputs"] = rewritten; + auto jsonSh = writeStructuredAttrsShell(json); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index f021e4de3..ee6bbb045 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -124,7 +124,7 @@ bool ParsedDerivation::substitutesAllowed() const } static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); -std::optional ParsedDerivation::prepareStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths) +std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); if (!structuredAttrs) return std::nullopt; @@ -134,14 +134,7 @@ std::optional ParsedDerivation::prepareStructuredAttrs(std::opti /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv.outputs) { - if (inputRewrites) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites.value()); - } else { - /* This case is only relevant for the nix-shell */ - outputs[i.first] = hashPlaceholder(i.first); - } + outputs[i.first] = hashPlaceholder(i.first); } json["outputs"] = outputs; diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 8061e3a8b..5e3fb93ca 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -37,7 +37,7 @@ public: bool substitutesAllowed() const; - std::optional prepareStructuredAttrs(std::optional inputRewrites, Store & store, const StorePathSet & inputPaths); + std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; std::string writeStructuredAttrsShell(nlohmann::json & json); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 459876465..519cbd5bb 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -444,7 +444,7 @@ static void main_nix_build(int argc, char * * argv) drv ); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs(std::nullopt, *store, inputs)) { + if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { auto json = structAttrs.value(); structuredAttrsRC = writeStructuredAttrsShell(json); From 8b6fba2b63820ea63599d200f79740bb7f85de1e Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 24 Jun 2021 15:44:13 +0200 Subject: [PATCH 113/555] Eventually delete the CA paths lock files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark the lockfiles as having to eventually be deleted so that they don’t stay laying around in the store at the end of the build Fix #4936 --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ba0aca29c..8320dd1c4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2481,6 +2481,7 @@ void LocalDerivationGoal::registerOutputs() floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); auto optFixedPath = output.path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) From 57409244ec8f568ce6b4ecb1bde0e509942ac05f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 24 Jun 2021 18:02:51 +0200 Subject: [PATCH 114/555] boehmgc: Crude support for coroutines Fixes the problem where a stack pointer outside the original thread causes the collector to crash. It could be made more accurate by recording the stack pointer every time we switch to a coroutine. We can use this information to update our own coroutine stacks like normal data. When the stack pointer is on a thread, we can add a field to GC_thread "fallback_sp" to be used when the thread sp is outside the original thread range. --- boehmgc-coroutine-sp-fallback.diff | 44 ++++++++++++++++++++++++++++++ flake.nix | 8 +++++- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 boehmgc-coroutine-sp-fallback.diff diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff new file mode 100644 index 000000000..c618718a7 --- /dev/null +++ b/boehmgc-coroutine-sp-fallback.diff @@ -0,0 +1,44 @@ +diff --git a/pthread_stop_world.c b/pthread_stop_world.c +index 1cee6a0b..8977b0dd 100644 +--- a/pthread_stop_world.c ++++ b/pthread_stop_world.c +@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void) + struct GC_traced_stack_sect_s *traced_stack_sect; + pthread_t self = pthread_self(); + word total_size = 0; ++ size_t stack_limit; ++ pthread_attr_t pattr; + + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); +@@ -723,6 +725,30 @@ GC_INNER void GC_push_all_stacks(void) + hi = p->altstack + p->altstack_size; + /* FIXME: Need to scan the normal stack too, but how ? */ + /* FIXME: Assume stack grows down */ ++ } else { ++ if (pthread_getattr_np(p->id, &pattr)) { ++ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); ++ } ++ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { ++ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); ++ } ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ #ifndef STACK_GROWS_UP ++ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack ++ lo = hi - stack_limit; ++ } ++ #else ++ if (lo < hi || lo >= hi + stack_limit) { // sp outside stack ++ lo = hi + stack_limit; ++ } ++ #endif + } + GC_push_all_stack_sections(lo, hi, traced_stack_sect); + # ifdef STACK_GROWS_UP diff --git a/flake.nix b/flake.nix index 6c65e3406..8e7dd75d3 100644 --- a/flake.nix +++ b/flake.nix @@ -102,7 +102,13 @@ }); propagatedDeps = - [ (boehmgc.override { enableLargeConfig = true; }) + [ ((boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + ]; + })) ]; perlDeps = From bf68c693dc5157c2be1f3a9f407dd1ce3761df78 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Jun 2021 11:17:19 +0200 Subject: [PATCH 115/555] tests: Get rid of some result symlinks Fixes error: cannot create symlink '/home/eelco/Dev/nix/tests/result'; already exists --- tests/build.sh | 6 +++--- tests/ca/duplicate-realisation-in-closure.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/build.sh b/tests/build.sh index ce9d6602c..c77f620f7 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -1,7 +1,7 @@ source common.sh expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' -nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status ' +nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' (.[0] | (.drvPath | match(".*multiple-outputs-a.drv")) and (.outputs.first | match(".*multiple-outputs-a-first")) and @@ -10,10 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status ' (.drvPath | match(".*multiple-outputs-b.drv")) and (.outputs.out | match(".*multiple-outputs-b"))) ' - testNormalization () { clearStore - outPath=$(nix-build ./simple.nix) + outPath=$(nix-build ./simple.nix --no-out-link) test "$(stat -c %Y $outPath)" -eq 1 } + testNormalization diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh index bfe2a4e08..ca9099641 100644 --- a/tests/ca/duplicate-realisation-in-closure.sh +++ b/tests/ca/duplicate-realisation-in-closure.sh @@ -17,10 +17,10 @@ sleep 2 # To make sure that `$(date)` will be different # As we’ve cleared the cache, we’ll have to rebuild current-time. And because # the current time isn’t the same as before, this will yield a new (different) # realisation -nix build -f nondeterministic.nix dep2 +nix build -f nondeterministic.nix dep2 --no-link # Build something that depends both on dep1 and dep2. # If everything goes right, we should rebuild dep2 rather than fetch it from # the cache (because that would mean duplicating `current-time` in the closure), # and have `dep1 == dep2`. -nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs +nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link From 5c58d84a76d96f269e3ff1e72c9c9ba5f68576af Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Jun 2021 17:45:48 +0200 Subject: [PATCH 116/555] boehmgc: Remove unused code from patch --- boehmgc-coroutine-sp-fallback.diff | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index c618718a7..fa8dd0325 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,5 +1,5 @@ diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 1cee6a0b..8977b0dd 100644 +index 1cee6a0b..46c3acd9 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c @@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void) @@ -11,7 +11,7 @@ index 1cee6a0b..8977b0dd 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -723,6 +725,30 @@ GC_INNER void GC_push_all_stacks(void) +@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ @@ -35,9 +35,7 @@ index 1cee6a0b..8977b0dd 100644 + lo = hi - stack_limit; + } + #else -+ if (lo < hi || lo >= hi + stack_limit) { // sp outside stack -+ lo = hi + stack_limit; -+ } ++ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." + #endif } GC_push_all_stack_sections(lo, hi, traced_stack_sect); From c906d6530dc52d79f69e65e0447893084a0d4556 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jun 2021 00:12:03 -0500 Subject: [PATCH 117/555] Support cross-compiling binaryTarball --- flake.nix | 164 ++++++++++++++++++---------------- maintainers/upload-release.pl | 2 + scripts/install.in | 10 +++ 3 files changed, 99 insertions(+), 77 deletions(-) diff --git a/flake.nix b/flake.nix index 829cd2804..08b385ec2 100644 --- a/flake.nix +++ b/flake.nix @@ -135,10 +135,11 @@ substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: - '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + (system: let + tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; + in '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) systems @@ -176,6 +177,77 @@ }; + binaryTarball = buildPackages: nix: pkgs: let + inherit (pkgs) cacert; + installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; + in + + buildPackages.runCommand "nix-binary-tarball-${version}" + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${pkgs.system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -315,85 +387,23 @@ # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: + binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system}); - with nixpkgsFor.${system}; - - let - installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; }; - in - - runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''); + binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: { + name = crossSystem; + value = let + nixpkgsCross = import nixpkgs { + inherit system crossSystem; + overlays = [ self.overlay ]; + }; + in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; + }) crossSystems)); # The first half of the installation script. This is uploaded # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index c2933300f..77007f914 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -110,6 +110,8 @@ downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); downloadFile("installerScript", "1"); for my $fn (glob "$tmpDir/*") { diff --git a/scripts/install.in b/scripts/install.in index 39016d161..e801d4268 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -40,6 +40,16 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-linux@ system=aarch64-linux ;; + Linux.armv6l_linux) + hash=@tarballHash_armv6l-linux@ + path=@tarballPath_armv6l-linux@ + system=armv6l-linux + ;; + Linux.armv7l_linux) + hash=@tarballHash_armv7l-linux@ + path=@tarballPath_armv7l-linux@ + system=armv7l-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ From 580583e0b386df181a3c30519c821275362b10e0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jun 2021 00:14:54 -0500 Subject: [PATCH 118/555] Build cross-compilation in gha --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 08b385ec2..8b37a5658 100644 --- a/flake.nix +++ b/flake.nix @@ -404,7 +404,7 @@ # tarball for the user's system and calls the second half of the # installation script. installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; # Line coverage analysis. coverage = From decc14d4b7fbe1664ae609563003c883d4e426a8 Mon Sep 17 00:00:00 2001 From: Florian Franzen Date: Mon, 28 Jun 2021 16:27:03 +0200 Subject: [PATCH 119/555] man: fix formatting of nix3-profile-remove --- src/nix/profile-remove.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index dcf825da9..ba85441d8 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -15,6 +15,7 @@ R""( ``` * Remove all packages: + ```console # nix profile remove '.*' ``` From 777c688a9875c1b1864246b28b128f5431cbc6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 28 Jun 2021 18:42:44 +0200 Subject: [PATCH 120/555] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/bb8a5e54845012ed1375ffd5f317d2fdf434b20e' -> 'github:NixOS/nixpkgs/f77036342e2b690c61c97202bf48f2ce13acc022' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8aad22957..5fc969d7b 100644 --- a/flake.lock +++ b/flake.lock @@ -19,11 +19,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1622593737, - "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=", + "lastModified": 1624862269, + "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e", + "rev": "f77036342e2b690c61c97202bf48f2ce13acc022", "type": "github" }, "original": { From 6c13a3f7354d02fb3a38604ed601b76c011e826f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 28 Jun 2021 15:08:17 -0500 Subject: [PATCH 121/555] Support binaryTarballCross in gha --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8b37a5658..c97130ac9 100644 --- a/flake.nix +++ b/flake.nix @@ -387,7 +387,7 @@ # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system}); + binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: { name = crossSystem; @@ -498,7 +498,10 @@ # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; } "touch $out"; - }); + } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: { + name = "binaryTarball-${crossSystem}"; + value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem}; + }) crossSystems)) else {})); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; From 9feca5cdf64b82bfb06dfda07d19d007a2dfa1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 28 Jun 2021 23:02:53 +0200 Subject: [PATCH 122/555] github actions: simplify getting the system logic --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17a79dc97..b2b1f07fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,8 +20,7 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - #- run: nix flake check - - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) + - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)') check_cachix: name: Cachix secret present for installer tests runs-on: ubuntu-latest From bf7960a4ede0f5ba7e88b939ed0f33aa6087cbc6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 29 Jun 2021 10:13:42 +0200 Subject: [PATCH 123/555] develop: Discard the input{Overrides,Updates} when getting bash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `nix develop` is getting bash from an (assumed existing) `nixpkgs` flake. However, when doing so, it reuses the `lockFlags` passed to the current flake, including the `--input-overrides` and `--input-update` which generally don’t make sense anymore at that point (and trigger a warning because of that) Clear these overrides before getting the nixpkgs flake to get rid of the warning. --- src/nix/develop.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d77ff52d7..6c089469d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,13 +443,17 @@ struct CmdDevelop : Common, MixEnvironment try { auto state = getEvalState(); + auto nixpkgsLockFlags = lockFlags; + nixpkgsLockFlags.inputOverrides = {}; + nixpkgsLockFlags.inputUpdates = {}; + auto bashInstallable = std::make_shared( this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, - lockFlags); + nixpkgsLockFlags); shell = state->store->printStorePath( toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; From 4cff8188a5b59303f726a2497727fff34f9711d3 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 29 Jun 2021 13:53:56 +0200 Subject: [PATCH 124/555] Keep the `isFlake` attribute for overriden inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When overriding an input that isn’t a flake, mark the override as also not being a flake. Fix #3774 --- src/libexpr/flake/flake.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2e94490d4..8e6f06949 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -359,7 +359,12 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); - if (hasOverride) overridesUsed.insert(inputPath); + if (hasOverride) { + overridesUsed.insert(inputPath); + // Respect the “flakeness” of the input even if we + // override it + i->second.isFlake = input2.isFlake; + } auto & input = hasOverride ? i->second : input2; /* Resolve 'follows' later (since it may refer to an input From 7daf0c6ef12a0fbeedb53ed683c93478764fe1ab Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 29 Jun 2021 14:52:46 +0200 Subject: [PATCH 125/555] Forward the experimental features to the nix repl subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass the current experimental features using `NIX_CONFIG` to the various Nix subprocesses that `nix repl` invokes. This is quite a hack, but having `nix repl` call Nix with a subprocess is a hack already, so I guess that’s fine. --- src/nix/repl.cc | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index eed79c332..0275feae7 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -104,6 +104,26 @@ NixRepl::~NixRepl() write_history(historyFile.c_str()); } +string runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto experimentalFeatures = concatStringsSep(" ", settings.experimentalFeatures.get()); + auto nixConf = getEnv("NIX_CONFIG").value_or(""); + nixConf.append("\nexperimental-features = " + experimentalFeatures); + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = nixConf; + RunOptions opts(settings.nixBinDir+ "/" + program, args); + opts.input = input; + opts.environment = subprocessEnv; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + static NixRepl * curRepl; // ugly static char * completionCallback(char * s, int *match) { @@ -463,7 +483,7 @@ bool NixRepl::processLine(string line) state->callFunction(f, v, result, Pos()); StorePath drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)}); + runNix("nix-shell", {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { @@ -477,7 +497,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ try { - runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw}); + runNix("nix", {"build", "--no-link", drvPathRaw}); auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputsAndOptPaths(*state->store)) @@ -485,9 +505,9 @@ bool NixRepl::processLine(string line) } catch (ExecError &) { } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw}); + runNix("nix-env", {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw}); + runNix("nix-shell", {drvPathRaw}); } } From 7351656b82e59e46610c7e9edee023615b153e0e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 29 Jun 2021 21:46:54 -0500 Subject: [PATCH 126/555] Only cross compile from x86_64-linux This is broken on aarch64-linux / x86_64-darwin, so might as well just disable it for now. --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 999028ec2..7d41947a7 100644 --- a/flake.nix +++ b/flake.nix @@ -385,7 +385,7 @@ buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem: - nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}."nix-${crossSystem}")); + nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); @@ -395,7 +395,7 @@ # the installation script. binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); - binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: { + binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: { name = crossSystem; value = let nixpkgsCross = import nixpkgs { From 2200f315daff989a974ed4fac445f90a06d55e81 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 29 Jun 2021 21:48:07 -0500 Subject: [PATCH 127/555] Disable -pie on static nix This should resolve the failing build. See https://github.com/NixOS/nixpkgs/pull/128674 for a better fix. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 7d41947a7..5430d3904 100644 --- a/flake.nix +++ b/flake.nix @@ -549,6 +549,8 @@ stripAllList = ["bin"]; strictDeps = true; + + hardeningDisable = [ "pie" ]; }; } // builtins.listToAttrs (map (crossSystem: { name = "nix-${crossSystem}"; From 5be17a4b96fa4dbd512da4cba59f177cdbd84935 Mon Sep 17 00:00:00 2001 From: Pamplemousse Date: Fri, 25 Jun 2021 15:57:37 -0700 Subject: [PATCH 128/555] Allow to compile after `./configure --enable-gc=no` Signed-off-by: Pamplemousse --- src/libexpr/eval.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ef9f8efca..ceca6e16b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -64,7 +64,11 @@ static char * dupStringWithLen(const char * s, size_t size) RootValue allocRootValue(Value * v) { +#if HAVE_BOEHMGC return std::allocate_shared(traceable_allocator(), v); +#else + return std::make_shared(v); +#endif } From 093ed4763615a73edb1e8567d2f41d55ff5aaa0c Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Wed, 30 Jun 2021 22:13:32 +0300 Subject: [PATCH 129/555] nix registry: add --registry flag --- src/nix/registry-add.md | 7 ++++ src/nix/registry-pin.md | 7 ++++ src/nix/registry-remove.md | 6 ++++ src/nix/registry.cc | 67 ++++++++++++++++++++++++++++++-------- 4 files changed, 73 insertions(+), 14 deletions(-) diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md index 80a31996a..a947fa0b3 100644 --- a/src/nix/registry-add.md +++ b/src/nix/registry-add.md @@ -21,6 +21,13 @@ R""( # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs ``` +* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake + registry: + + ```console + nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs + ``` + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index 6e97e003e..7b163c463 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -24,6 +24,13 @@ R""( … ``` +* Pin `nixpkgs` in a custom registry to its most recent Git revision + + ```console + # nix registry pin --registry ./custom-flake-registry.json nixpkgs + ``` + + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md index 4c0eb4947..eecd4c6e7 100644 --- a/src/nix/registry-remove.md +++ b/src/nix/registry-remove.md @@ -8,6 +8,12 @@ R""( # nix registry remove nixpkgs ``` +* Remove the entry `nixpkgs` from a custom registry: + + ```console + # nix registry remove --registry ./custom-flake-registry.json nixpkgs + ``` + # Description This command removes from the user registry any entry for flake diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f9719600f..4400cc8dc 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -10,6 +10,45 @@ using namespace nix; using namespace nix::flake; + +class RegistryCommand: virtual Args +{ + std::string registry_path; + + std::shared_ptr registry; + + +public: + + RegistryCommand() + { + addFlag({ + .longName = "registry", + .description = "The registry to operate on.", + .labels = {"registry"}, + .handler = {®istry_path}, + }); + } + + std::shared_ptr getRegistry() { + if (registry) return registry; + if (registry_path.empty()) { + registry = fetchers::getUserRegistry(); + } else { + registry = fetchers::getCustomRegistry(registry_path); + } + return registry; + } + + Path getRegistryPath() { + if (registry_path.empty()) { + return fetchers::getUserRegistryPath(); + } else { + return registry_path; + } + } +}; + struct CmdRegistryList : StoreCommand { std::string description() override @@ -45,7 +84,7 @@ struct CmdRegistryList : StoreCommand } }; -struct CmdRegistryAdd : MixEvalArgs, Command +struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand { std::string fromUrl, toUrl; @@ -71,16 +110,16 @@ struct CmdRegistryAdd : MixEvalArgs, Command { auto fromRef = parseFlakeRef(fromUrl); auto toRef = parseFlakeRef(toUrl); + auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(fromRef.input); - userRegistry->add(fromRef.input, toRef.input, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->remove(fromRef.input); + registry->add(fromRef.input, toRef.input, extraAttrs); + registry->write(getRegistryPath()); } }; -struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command +struct CmdRegistryRemove : RegistryCommand, Command { std::string url; @@ -103,13 +142,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command void run() override { - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(parseFlakeRef(url).input); - userRegistry->write(fetchers::getUserRegistryPath()); + auto registry = getRegistry(); + registry->remove(parseFlakeRef(url).input); + registry->write(getRegistryPath()); } }; -struct CmdRegistryPin : virtual Args, EvalCommand +struct CmdRegistryPin : RegistryCommand, EvalCommand { std::string url; @@ -132,14 +171,14 @@ struct CmdRegistryPin : virtual Args, EvalCommand void run(nix::ref store) override { + auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(ref.input); + registry->remove(ref.input); auto [tree, resolved] = ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; - userRegistry->add(ref.input, resolved, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->add(ref.input, resolved, extraAttrs); + registry->write(getRegistryPath()); } }; From 811f3e8605ce779830fa4770cd7ef3cda81afdf1 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Wed, 30 Jun 2021 22:14:41 +0300 Subject: [PATCH 130/555] nix registry pin: add a way to pin to a custom locked --- src/libfetchers/registry.cc | 7 +++++++ src/libfetchers/registry.hh | 3 +++ src/nix/registry.cc | 19 +++++++++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 74376adc0..c2dfa9dc6 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -124,6 +124,13 @@ std::shared_ptr getUserRegistry() return userRegistry; } +std::shared_ptr getCustomRegistry(Path p) +{ + static auto customRegistry = + Registry::read(p, Registry::Custom); + return customRegistry; +} + static std::shared_ptr flagRegistry = std::make_shared(Registry::Flag); diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 1077af020..cde22dff6 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -14,6 +14,7 @@ struct Registry User = 1, System = 2, Global = 3, + Custom = 4, }; RegistryType type; @@ -48,6 +49,8 @@ typedef std::vector> Registries; std::shared_ptr getUserRegistry(); +std::shared_ptr getCustomRegistry(Path p); + Path getUserRegistryPath(); Registries getRegistries(ref store); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 4400cc8dc..ec80dc0bc 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -152,9 +152,11 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { std::string url; + std::string locked; + std::string description() override { - return "pin a flake to its current version in user flake registry"; + return "pin a flake to its current version in user flake registry or to the current version of a flake URI"; } std::string doc() override @@ -167,14 +169,27 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand CmdRegistryPin() { expectArg("url", &url); + + expectArgs({ + .label = "locked", + .optional = true, + .handler = {&locked}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); } void run(nix::ref store) override { + if (locked.empty()) { + locked = url; + } auto registry = getRegistry(); auto ref = parseFlakeRef(url); + auto locked_ref = parseFlakeRef(locked); registry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input.fetch(store); + auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); From ef1e7ab840eaa50da29ddde7bf990a1ff6a8b185 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 1 Jul 2021 00:23:47 +0300 Subject: [PATCH 131/555] flake.nixConfig: fix flake-registry config settings Before this commit, nixConfig.flake-registry didn't have any real effect on the evaluation, since config was applied after inputs were evaluated. Change this behavior: apply the config in the beginning of flake::lockFile. --- src/libcmd/installables.cc | 2 -- src/libexpr/flake/flake.cc | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index fe52912cf..540e53eee 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -575,8 +575,6 @@ std::shared_ptr InstallableFlake::getLockedFlake() const { if (!_lockedFlake) { _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); - _lockedFlake->flake.config.apply(); - // FIXME: send new config to the daemon. } return _lockedFlake; } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8e6f06949..b57679d3b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -298,6 +298,9 @@ LockedFlake lockFlake( auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); + flake.config.apply(); + // FIXME: send new config to the daemon. + try { // FIXME: symlink attack From e756a59c72e3e793ed01dec04749fa356f263315 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 1 Jul 2021 17:54:22 +0300 Subject: [PATCH 132/555] fixup! flake.nixConfig: fix flake-registry config settings --- src/libcmd/installables.cc | 4 +++- src/libexpr/flake/flake.cc | 6 ++++-- src/libexpr/flake/flake.hh | 4 ++++ src/nix/flake.cc | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 540e53eee..269be11a2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -573,8 +573,10 @@ InstallableFlake::getCursors(EvalState & state) std::shared_ptr InstallableFlake::getLockedFlake() const { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); + _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b57679d3b..e266bc36d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -298,8 +298,10 @@ LockedFlake lockFlake( auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); - flake.config.apply(); - // FIXME: send new config to the daemon. + if (lockFlags.applyNixConfig) { + flake.config.apply(); + // FIXME: send new config to the daemon. + } try { diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index d17d5e183..4479e95db 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -104,6 +104,10 @@ struct LockFlags references like 'nixpkgs'. */ bool useRegistries = true; + /* Whether to apply flake's nixConfig attribute to the configuration */ + + bool applyNixConfig = false; + /* Whether mutable flake references (i.e. those without a Git revision or similar) without a corresponding lock are allowed. Mutable flake references with a lock are always diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 64fcfc000..9055b07eb 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -84,6 +84,7 @@ struct CmdFlakeUpdate : FlakeCommand lockFlags.recreateLockFile = true; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -114,6 +115,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -270,6 +272,8 @@ struct CmdFlakeCheck : FlakeCommand settings.readOnlyMode = !build; auto state = getEvalState(); + + lockFlags.applyNixConfig = true; auto flake = lockFlake(); bool hasErrors = false; From 4a7a8b87cd0dd383f1059fe7903c4b001314f58d Mon Sep 17 00:00:00 2001 From: Pamplemousse Date: Tue, 29 Jun 2021 14:28:43 -0700 Subject: [PATCH 133/555] Prefer to throw specific errors Signed-off-by: Pamplemousse --- src/libcmd/command.cc | 2 +- src/libexpr/attr-path.cc | 6 +++--- src/libexpr/eval.cc | 2 +- src/libutil/url.cc | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 569c4b9e4..6777b23be 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -188,7 +188,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) } if (result.size() != 1) - throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); + throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); updateProfile(result[0]); } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9dd557205..867eb13a5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s) ++i; while (1) { if (i == s.end()) - throw Error("missing closing quote in selection path '%1%'", s); + throw ParseError("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto colon = pos.rfind(':'); if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); + throw ParseError("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); unsigned int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); + throw ParseError("cannot parse line number '%s'", pos); } Symbol file = state.symbols.create(filename); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ef9f8efca..657e4e4cf 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -908,7 +908,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) // computation. if (mustBeTrivial && !(dynamic_cast(e))) - throw Error("file '%s' must be an attribute set", path); + throw EvalError("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", path2); diff --git a/src/libutil/url.cc b/src/libutil/url.cc index c1bab866c..f6232d255 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url) auto isFile = scheme.find("file") != std::string::npos; if (authority && *authority != "" && isFile) - throw Error("file:// URL '%s' has unexpected authority '%s'", + throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); if (isFile && path.empty()) From 20cce079f258dc3f62340ecd7b9de1bbd99ef814 Mon Sep 17 00:00:00 2001 From: "Yestin L. Harrison" Date: Thu, 1 Jul 2021 18:19:01 -0600 Subject: [PATCH 134/555] Respect TERM=dumb more consistently --- src/libmain/progress-bar.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/util.cc | 4 ++++ src/libutil/util.hh | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 15354549a..b2a6e2a82 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -484,7 +484,7 @@ Logger * makeProgressBar(bool printBuildLogs) { return new ProgressBar( printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + shouldANSI() ); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d2e801175..6b9b850ca 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -46,7 +46,7 @@ public: : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; - tty = isatty(STDERR_FILENO); + tty = shouldANSI(); } bool isVerbose() override { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7e57fd7ca..94460b367 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1372,6 +1372,10 @@ void ignoreException() } } +bool shouldANSI() +{ + return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"; +} std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f84d0fb31..a8dd4bd47 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -482,6 +482,9 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; +/* Determine whether ANSI escape sequences are appropriate for the + present output. */ +bool shouldANSI(); /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, From 74838deeb8b9c92fb9aa8abfb53d3f7f31db5f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 Jul 2021 14:12:00 +0200 Subject: [PATCH 135/555] upload-release.pl: add aarch64-darwin --- maintainers/upload-release.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77007f914..3a8d27499 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -110,6 +110,7 @@ downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarball.aarch64-darwin", "1"); downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); downloadFile("installerScript", "1"); @@ -155,6 +156,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . "}\n"); system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; From d8ad6f1c108ec253b307c30aadee460cf602dbd1 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 2 Jul 2021 16:10:57 +0300 Subject: [PATCH 136/555] Add tests for --registry and second arg for nix registry pin --- tests/flakes.sh | 80 +++++++------------------------------------------ 1 file changed, 10 insertions(+), 70 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 9764e1a6c..a4e657980 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -90,76 +90,14 @@ EOF git -C $nonFlakeDir add README.md git -C $nonFlakeDir commit -m 'Initial' -cat > $registry < Date: Fri, 2 Jul 2021 09:33:54 -0600 Subject: [PATCH 137/555] Add $NO_COLOR check to ANSI escape conditions --- src/libutil/util.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 94460b367..ee9f17228 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1374,7 +1374,9 @@ void ignoreException() bool shouldANSI() { - return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"; + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !getEnv("NO_COLOR").has_value(); } std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) From ec2c6bd47041bda1f30e66ee48de662ddcd07378 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 2 Jul 2021 19:20:07 +0200 Subject: [PATCH 138/555] Allow scp-style uris in `fetchgit` Fix #5303 --- src/libexpr/primops/fetchTree.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b8b99d4fa..c593400a7 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -60,10 +61,19 @@ void emitTreeAttrs( v.attrs->sort(); } -std::string fixURI(std::string uri, EvalState &state) +std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : "file://" + uri; + return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; +} + +std::string fixURIForGit(std::string uri, EvalState & state) +{ + static std::regex scp_uri("([^/].*)@(.*):(.*)"); + if (uri[0] != '/' && std::regex_match(uri, scp_uri)) + return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); + else + return fixURI(uri, state); } void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) @@ -121,15 +131,15 @@ static void fetchTree( input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + auto url = state.coerceToString(pos, *args[0], context, false, false); if (type == "git") { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", url); + attrs.emplace("url", fixURIForGit(url, state)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(fixURI(url, state)); } } From 70cb2ffaccb91eff6f4afe2552d0784e279f1fe9 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 3 Jul 2021 14:19:10 +0200 Subject: [PATCH 139/555] libcmd/installables: implement completion for Nix expressions passed via `-f` --- src/libcmd/installables.cc | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index fe52912cf..49f063334 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -171,14 +171,44 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { - if (file) return; // FIXME + if (file) { + evalSettings.pureEval = false; + auto state = getEvalState(); + Expr *e = state->parseExprFromFile( + resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) + ); - completeFlakeRefWithFragment( - getEvalState(), - lockFlags, - getDefaultFlakeAttrPathPrefixes(), - getDefaultFlakeAttrPaths(), - prefix); + Value root; + state->eval(e, root); + + auto autoArgs = getAutoArgs(*state); + + std::string prefix_ = std::string(prefix); + auto sep = prefix_.rfind('.'); + if (sep != std::string::npos) { + prefix_.erase(sep); + } else { + prefix_ = ""; + } + + Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first); + state->forceValue(v1); + Value v2; + state->autoCallFunction(*autoArgs, v1, v2); + + if (v2.type() == nAttrs) { + for (auto & i : *v2.attrs) { + completions->add(i.name); + } + } + } else { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); + } } void completeFlakeRefWithFragment( From 087c5f5325c46485a9bc5f8e7f2620af6bf6bf56 Mon Sep 17 00:00:00 2001 From: Michael Fellinger Date: Mon, 5 Jul 2021 12:00:08 +0200 Subject: [PATCH 140/555] Fix devShell handling of env values including @ and % --- src/nix/develop.cc | 2 +- tests/shell.nix | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6c089469d..699ec0b99 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -54,7 +54,7 @@ BuildEnvironment readEnvironment(const Path & path) R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re"; static std::string dquotedStringRegex = R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; diff --git a/tests/shell.nix b/tests/shell.nix index 53759f99a..f174db583 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -34,6 +34,8 @@ let pkgs = rec { name = "shellDrv"; builder = "/does/not/exist"; VAR_FROM_NIX = "bar"; + ASCII_PERCENT = "%"; + ASCII_AT = "@"; TEST_inNixShell = if inNixShell then "true" else "false"; inherit stdenv; outputs = ["dev" "out"]; From c8a80e4dbe9c743ac0280a2504a0d451e8848b28 Mon Sep 17 00:00:00 2001 From: Jarrett Keifer Date: Mon, 5 Jul 2021 07:48:53 -0700 Subject: [PATCH 141/555] Fix wrong hash var for aarch64-darwin binary --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index e801d4268..ffc1f2785 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -56,7 +56,7 @@ case "$(uname -s).$(uname -m)" in system=x86_64-darwin ;; Darwin.arm64|Darwin.aarch64) - hash=@binaryTarball_aarch64-darwin@ + hash=@tarballHash_aarch64-darwin@ path=@tarballPath_aarch64-darwin@ system=aarch64-darwin ;; From 3c5f69bb60f858f471121adaf172edb47628188e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 5 Jul 2021 21:37:33 +0200 Subject: [PATCH 142/555] completeInstallable: also match for already typed prefixes --- src/libcmd/installables.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 49f063334..5f263061b 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -185,9 +185,12 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string prefix_ = std::string(prefix); auto sep = prefix_.rfind('.'); + std::string searchWord; if (sep != std::string::npos) { - prefix_.erase(sep); + searchWord = prefix_.substr(sep, std::string::npos); + prefix_ = prefix_.substr(0, sep); } else { + searchWord = prefix_; prefix_ = ""; } @@ -198,7 +201,10 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) if (v2.type() == nAttrs) { for (auto & i : *v2.attrs) { - completions->add(i.name); + std::string name = i.name; + if (name.find(searchWord) == 0) { + completions->add(i.name); + } } } } else { From 83615fcf8fe6095b60e33097e2d15fb1a3f56298 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 5 Jul 2021 13:09:46 +0200 Subject: [PATCH 143/555] Allow `fetchGit` to take a `name` argument Fix #3388 --- src/libfetchers/fetchers.cc | 7 ++++++- src/libfetchers/fetchers.hh | 2 ++ src/libfetchers/git.cc | 8 ++++---- tests/fetchGit.sh | 4 ++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 916e0a8e8..e158d914b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -200,12 +200,17 @@ void Input::markChangedFile( return scheme->markChangedFile(*this, file, commitMsg); } +std::string Input::getName() const +{ + return maybeGetStrAttr(attrs, "name").value_or("source"); +} + StorePath Input::computeStorePath(Store & store) const { auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); } std::string Input::getType() const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a72cfafa4..c839cf23b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -81,6 +81,8 @@ public: std::string_view file, std::optional commitMsg) const; + std::string getName() const; + StorePath computeStorePath(Store & store) const; // Convenience functions for common attributes. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d8e0dbe0a..bc1930138 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -60,7 +60,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -167,10 +167,10 @@ struct GitInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { - auto name = "source"; - Input input(_input); + std::string name = input.getName(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); @@ -270,7 +270,7 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 88744ee7f..89294d8d2 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -189,3 +189,7 @@ path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rep rev4=$(git -C $repo rev-parse HEAD) rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev") [[ $rev4 = $rev4_nix ]] + +# The name argument should be handled +path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath") +[[ $path9 =~ -foo$ ]] From a487a652ed2f662782529d8f2bea4cf90a05e0c9 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 6 Jul 2021 08:42:47 +0200 Subject: [PATCH 144/555] allow fetchTarball to take a `name` argument --- src/libfetchers/tarball.cc | 4 ++-- tests/tarball.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 257465bae..031ccc5f7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -196,7 +196,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") throw Error("unsupported tarball input attribute '%s'", name); Input input; @@ -226,7 +226,7 @@ struct TarballInputScheme : InputScheme std::pair fetch(ref store, const Input & input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; return {std::move(tree), input}; } }; diff --git a/tests/tarball.sh b/tests/tarball.sh index d53ec8cd9..279ef5672 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -30,6 +30,7 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 From e4b082a52b4cd983b50037783a647cf39cd18bdf Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 6 Jul 2021 08:43:06 +0200 Subject: [PATCH 145/555] allow fetchMercurial to take a `name` argument --- src/libexpr/primops/fetchMercurial.cc | 1 + src/libfetchers/mercurial.cc | 8 ++++---- tests/fetchMercurial.sh | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 4830ebec3..3f88ccb91 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar fetchers::Attrs attrs; attrs.insert_or_assign("type", "hg"); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + attrs.insert_or_assign("name", name); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 0eb401e10..efb4ee8db 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -74,7 +74,7 @@ struct MercurialInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "hg") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") throw Error("unsupported Mercurial input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -147,10 +147,10 @@ struct MercurialInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { - auto name = "source"; - Input input(_input); + auto name = input.getName(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug @@ -193,7 +193,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return { Tree(store->toRealPath(storePath), std::move(storePath)), diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index d8a4e09d2..726840664 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -94,3 +94,8 @@ hg commit --cwd $repo -m 'Bla3' path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $path2 = $path4 ]] + +echo paris > $repo/hello +# Passing a `name` argument should be reflected in the output path +path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath") +[[ $path5 =~ -foo$ ]] From 2c8240677edb94f17cc0bcd681e442c46b987013 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 6 Jul 2021 08:43:19 +0200 Subject: [PATCH 146/555] allow fetchFromGitHub to take a `name` argument --- src/libfetchers/github.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8352ef02d..298c05f9a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); From 3b3e6bb1e50fae42ab509b0e3910c93b83dc84cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Jul 2021 10:02:55 +0200 Subject: [PATCH 147/555] Style tweaks --- src/libfetchers/registry.cc | 2 +- src/libfetchers/registry.hh | 2 +- src/nix/registry-pin.md | 2 +- src/nix/registry.cc | 11 ++++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index c2dfa9dc6..f35359d4b 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -124,7 +124,7 @@ std::shared_ptr getUserRegistry() return userRegistry; } -std::shared_ptr getCustomRegistry(Path p) +std::shared_ptr getCustomRegistry(const Path & p) { static auto customRegistry = Registry::read(p, Registry::Custom); diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index cde22dff6..260a2c460 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -49,7 +49,7 @@ typedef std::vector> Registries; std::shared_ptr getUserRegistry(); -std::shared_ptr getCustomRegistry(Path p); +std::shared_ptr getCustomRegistry(const Path & p); Path getUserRegistryPath(); diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index 7b163c463..ebc0e3eff 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -24,7 +24,7 @@ R""( … ``` -* Pin `nixpkgs` in a custom registry to its most recent Git revision +* Pin `nixpkgs` in a custom registry to its most recent Git revision: ```console # nix registry pin --registry ./custom-flake-registry.json nixpkgs diff --git a/src/nix/registry.cc b/src/nix/registry.cc index ec80dc0bc..6a92576c7 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -11,13 +11,12 @@ using namespace nix; using namespace nix::flake; -class RegistryCommand: virtual Args +class RegistryCommand : virtual Args { std::string registry_path; std::shared_ptr registry; - public: RegistryCommand() @@ -30,7 +29,8 @@ public: }); } - std::shared_ptr getRegistry() { + std::shared_ptr getRegistry() + { if (registry) return registry; if (registry_path.empty()) { registry = fetchers::getUserRegistry(); @@ -40,7 +40,8 @@ public: return registry; } - Path getRegistryPath() { + Path getRegistryPath() + { if (registry_path.empty()) { return fetchers::getUserRegistryPath(); } else { @@ -156,7 +157,7 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand std::string description() override { - return "pin a flake to its current version in user flake registry or to the current version of a flake URI"; + return "pin a flake to its current version or to the current version of a flake URL"; } std::string doc() override From 02dd6bb610e55a009cd7a4c83639698d3a7acaa2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Jul 2021 10:48:47 +0200 Subject: [PATCH 148/555] tests/check.sh: Fix a race Fixes this random failure: error: hash mismatch in fixed-output derivation '/tmp/nix-shell.EUgAVU/nix-test/tests/check/store/sfps3l3c5n7dabpx34kigxnfhmrwk2h6-dummy.drv': specified: sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= got: sha256-0qhPS4tlCTfsj3PNi+LHSt1akRumTfJ0WO2CKdqASiY= which happens because multiple tests were writing to ./dummy. --- tests/add.sh | 2 +- tests/check.nix | 2 +- tests/check.sh | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/add.sh b/tests/add.sh index e26e05843..5c3eed793 100644 --- a/tests/add.sh +++ b/tests/add.sh @@ -9,7 +9,7 @@ echo $path2 if test "$path1" != "$path2"; then echo "nix-store --add and --add-fixed mismatch" exit 1 -fi +fi path3=$(nix-store --add-fixed sha256 ./dummy) echo $path3 diff --git a/tests/check.nix b/tests/check.nix index bca04fdaf..ec455ae2d 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -44,7 +44,7 @@ with import ./config.nix; }; hashmismatch = import { - url = "file://" + toString ./dummy; + url = "file://" + builtins.getEnv "TMPDIR" + "/dummy"; sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; }; diff --git a/tests/check.sh b/tests/check.sh index 5f4997e28..d26d4d8fc 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -74,12 +74,13 @@ nix-build check.nix -A fetchurl --no-out-link --check nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] +echo 'Hello World' > $TMPDIR/dummy nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] -echo -n > ./dummy +echo -n > $TMPDIR/dummy nix-build check.nix -A hashmismatch --no-out-link -echo 'Hello World' > ./dummy +echo 'Hello World' > $TMPDIR/dummy nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] From 6060ea1b0e4346a632b218d74b68ab18c20e100e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Jul 2021 11:30:35 +0200 Subject: [PATCH 149/555] Add tests/dummy --- .gitignore | 1 - tests/dummy | 1 + tests/init.sh | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 tests/dummy diff --git a/.gitignore b/.gitignore index 2e14561fe..5f402dbc3 100644 --- a/.gitignore +++ b/.gitignore @@ -76,7 +76,6 @@ perl/Makefile.config # /tests/ /tests/test-tmp /tests/common.sh -/tests/dummy /tests/result* /tests/restricted-innocent /tests/shell diff --git a/tests/dummy b/tests/dummy new file mode 100644 index 000000000..557db03de --- /dev/null +++ b/tests/dummy @@ -0,0 +1 @@ +Hello World diff --git a/tests/init.sh b/tests/init.sh index 1a6ccb6fe..6e45a939f 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -35,5 +35,3 @@ nix-store --init # Did anything happen? test -e "$NIX_STATE_DIR"/db/db.sqlite - -echo 'Hello World' > ./dummy From 7e5c79a2d210e7b986803d5fe5c7ebb504b3e74e Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Jul 2021 13:29:58 +0200 Subject: [PATCH 150/555] Forbid the `name` attribute for fetchTree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to support it for the “old” fetch* functions for backwards compatibility, but we don’t need it for fetchTree (as it’s a new function). Given that changing the `name` messes-up the content hashing, we can just forbid passing a custom `name` argument to it --- src/libexpr/primops/fetchTree.cc | 7 +++++++ tests/tarball.sh | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b8b99d4fa..472efe0ee 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -119,6 +119,13 @@ static void fetchTree( .errPos = pos }); + if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) + throw Error({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = pos + }); + + input = fetchers::Input::fromAttrs(std::move(attrs)); } else { auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); diff --git a/tests/tarball.sh b/tests/tarball.sh index 279ef5672..1301922a5 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -30,7 +30,6 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 @@ -41,6 +40,11 @@ test_tarball() { (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) nix-instantiate --eval -E '' -I fnord=file://no-such-tarball$ext -I fnord=. + + # Ensure that the `name` attribute isn’t accepted as that would mess + # with the content-addressing + (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }") + } test_tarball '' cat From a654c1d81c63f13a49591436d6638d7c42222b99 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Jul 2021 14:57:48 +0200 Subject: [PATCH 151/555] Restore the possibility to use a `name` parameter in fetchGit Accidentally broken by 7e5c79a2d210e7b986803d5fe5c7ebb504b3e74e --- src/libexpr/primops/fetchTree.cc | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 472efe0ee..a57bda04e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -72,13 +72,18 @@ void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v attrs.emplace(name, n == "url" ? fixURI(v, state) : v); } +struct FetchTreeParams { + bool emptyRevFallback = false; + bool allowNameArgument = false; +}; + static void fetchTree( EvalState &state, const Pos &pos, Value **args, Value &v, const std::optional type, - bool emptyRevFallback = false + const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; PathSet context; @@ -119,11 +124,12 @@ static void fetchTree( .errPos = pos }); - if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) - throw Error({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), - .errPos = pos - }); + if (!params.allowNameArgument) + if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) + throw Error({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = pos + }); input = fetchers::Input::fromAttrs(std::move(attrs)); @@ -151,13 +157,13 @@ static void fetchTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v, emptyRevFallback); + emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) { settings.requireExperimentalFeature("flakes"); - fetchTree(state, pos, args, v, std::nullopt); + fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } // FIXME: document @@ -299,7 +305,7 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) { - fetchTree(state, pos, args, v, "git", true); + fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } static RegisterPrimOp primop_fetchGit({ From b1cfe8f984145c836c1dab229c1117d6396a4305 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Jul 2021 17:12:01 +0200 Subject: [PATCH 152/555] nix develop: Add a test for arrays --- tests/nix-shell.sh | 1 + tests/shell.nix | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 3481c2c69..aae7cbcde 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -98,3 +98,4 @@ nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo # Test 'nix print-dev-env'. source <(nix print-dev-env -f shell.nix shellDrv) [[ -n $stdenv ]] +[[ ${arr1[2]} = "3 4" ]] diff --git a/tests/shell.nix b/tests/shell.nix index f174db583..42d2d2033 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -20,6 +20,7 @@ let pkgs = rec { for pkg in $buildInputs; do export PATH=$PATH:$pkg/bin done + declare -a arr1=(1 2 "3 4" 5) ''; stdenv = mkDerivation { From b1f1347ade81d1f04f2d490baceefb3c4de0b4e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 00:47:57 +0200 Subject: [PATCH 153/555] nix develop: Don't parse bash environment with regexes Instead have get-env.sh dump the bash environment as JSON. This should be a lot less error-prone. Fixes #4992. --- src/nix/develop.cc | 176 ++++++++++++++++++++++----------------------- src/nix/get-env.sh | 92 +++++++++++++++++++++++- 2 files changed, 174 insertions(+), 94 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 699ec0b99..e00f0d575 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,7 @@ #include "affinity.hh" #include "progress-bar.hh" -#include +#include using namespace nix; @@ -25,94 +25,98 @@ static DevelopSettings developSettings; static GlobalConfig::Register rDevelopSettings(&developSettings); -struct Var -{ - bool exported = true; - bool associative = false; - std::string quoted; // quoted string or array -}; - struct BuildEnvironment { - std::map env; - std::string bashFunctions; -}; + struct String + { + bool exported; + std::string value; + }; -BuildEnvironment readEnvironment(const Path & path) -{ - BuildEnvironment res; + using Array = std::vector; - std::set exported; + using Associative = std::map; - debug("reading environment file '%s'", path); + using Value = std::variant; - auto file = readFile(path); + std::map vars; + std::map bashFunctions; - auto pos = file.cbegin(); + static BuildEnvironment fromJSON(const Path & path) + { + BuildEnvironment res; - static std::string varNameRegex = - R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + std::set exported; - static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re"; + debug("reading environment file '%s'", path); - static std::string dquotedStringRegex = - R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + auto json = nlohmann::json::parse(readFile(path)); - static std::string squotedStringRegex = - R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; - - static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; - - static std::regex declareRegex( - "^declare -a?x (" + varNameRegex + ")(=(" + - dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); - - static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); - - /* Note: we distinguish between an indexed and associative array - using the space before the closing parenthesis. Will - undoubtedly regret this some day. */ - static std::regex assocArrayRegex( - "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); - - static std::regex functionRegex( - "^" + varNameRegex + " \\(\\) *\n"); - - while (pos != file.end()) { - - std::smatch match; - - if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - exported.insert(match[1]); + for (auto & [name, info] : json["variables"].items()) { + std::string type = info["type"]; + if (type == "var" || type == "exported") + res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }}); + else if (type == "array") + res.vars.insert({name, (Array) info["value"]}); + else if (type == "associative") + res.vars.insert({name, (Associative) info["value"]}); } - else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }}); + for (auto & [name, def] : json["bashFunctions"].items()) { + res.bashFunctions.insert({name, def}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }}); - } - - else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { - res.bashFunctions = std::string(pos, file.cend()); - break; - } - - else throw Error("shell environment '%s' has unexpected line '%s'", - path, file.substr(pos - file.cbegin(), 60)); + return res; } - res.env.erase("__output"); + void toBash(std::ostream & out, const std::set & ignoreVars) const + { + for (auto & [name, value] : vars) { + if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) { + if (auto str = std::get_if(&value)) { + out << fmt("%s=%s\n", name, shellEscape(str->value)); + if (str->exported) + out << fmt("export %s\n", name); + } + else if (auto arr = std::get_if(&value)) { + out << "declare -a " << name << "=("; + for (auto & s : *arr) + out << shellEscape(s) << " "; + out << ")\n"; + } + else if (auto arr = std::get_if(&value)) { + out << "declare -A " << name << "=("; + for (auto & [n, v] : *arr) + out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << ")\n"; + } + } + } - return res; -} + for (auto & [name, def] : bashFunctions) { + out << name << " ()\n{\n" << def << "}\n"; + } + } + + static std::string getString(const Value & value) + { + if (auto str = std::get_if(&value)) + return str->value; + else + throw Error("bash variable is not a string"); + } + + static Array getStrings(const Value & value) + { + if (auto str = std::get_if(&value)) + return tokenizeString(str->value); + else if (auto arr = std::get_if(&value)) { + return *arr; + } + else + throw Error("bash variable is not a string or array"); + } +}; const static std::string getEnvSh = #include "get-env.sh.gen.hh" @@ -185,7 +189,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) struct Common : InstallableCommand, MixProfile { - std::set ignoreVars{ + std::set ignoreVars{ "BASHOPTS", "EUID", "HOME", // FIXME: don't ignore in pure mode? @@ -233,22 +237,10 @@ struct Common : InstallableCommand, MixProfile out << "nix_saved_PATH=\"$PATH\"\n"; - for (auto & i : buildEnvironment.env) { - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted); - else { - out << fmt("%s=%s\n", i.first, i.second.quoted); - if (i.second.exported) - out << fmt("export %s\n", i.first); - } - } - } + buildEnvironment.toBash(out, ignoreVars); out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); @@ -258,16 +250,16 @@ struct Common : InstallableCommand, MixProfile auto script = out.str(); /* Substitute occurrences of output paths. */ - auto outputs = buildEnvironment.env.find("outputs"); - assert(outputs != buildEnvironment.env.end()); + auto outputs = buildEnvironment.vars.find("outputs"); + assert(outputs != buildEnvironment.vars.end()); // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : tokenizeString>(replaceStrings(outputs->second.quoted, "'", ""))) { - auto from = buildEnvironment.env.find(outputName); - assert(from != buildEnvironment.env.end()); + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); // FIXME: unquote - rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); } /* Substitute redirects. */ @@ -321,7 +313,7 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {readEnvironment(strPath), strPath}; + return {BuildEnvironment::fromJSON(strPath), strPath}; } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 091c0f573..834b84e35 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -8,10 +8,98 @@ if [[ -n $stdenv ]]; then source $stdenv/setup fi +# Better to use compgen, but stdenv bash doesn't have it. +__vars="$(declare -p)" +__functions="$(declare -F)" + +__dumpEnv() { + printf '{\n' + + printf ' "bashFunctions": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi + __fun_name="${BASH_REMATCH[1]}" + __fun_body="$(type $__fun_name)" + if [[ $__fun_body =~ \{(.*)\} ]]; then + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + __fun_body="${BASH_REMATCH[1]}" + printf " " + __escapeString "$__fun_name" + printf ':' + __escapeString "$__fun_body" + else + printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2 + return 1 + fi + done < <(printf "%s\n" "$__functions") + printf '\n },\n' + + printf ' "variables": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi + local type="${BASH_REMATCH[1]}" + local __var_name="${BASH_REMATCH[2]}" + + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + + printf " " + __escapeString "$__var_name" + printf ': {' + + # FIXME: handle -i, -r, -n. + if [[ $type == -x ]]; then + printf '"type": "exported", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -- ]]; then + printf '"type": "var", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -a ]]; then + printf '"type": "array", "value": [' + local __first2=1 + __var_name="$__var_name[@]" + for __i in "${!__var_name}"; do + if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi + __escapeString "$__i" + printf ' ' + done + printf ']' + elif [[ $type == -A ]]; then + printf '"type": "associative", "value": {\n' + local __first2=1 + declare -n __var_name2="$__var_name" + for __i in "${!__var_name2[@]}"; do + if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi + printf " " + __escapeString "$__i" + printf ": " + __escapeString "${__var_name2[$__i]}" + done + printf '\n }' + else + printf '"type": "unknown"' + fi + + printf "}" + done < <(printf "%s\n" "$__vars") + printf '\n }\n}' +} + +__escapeString() { + local __s="$1" + __s="${__s//\\/\\\\}" + __s="${__s//\"/\\\"}" + __s="${__s//$'\n'/\\n}" + __s="${__s//$'\r'/\\r}" + __s="${__s//$'\t'/\\t}" + printf '"%s"' "$__s" +} + +# Dump the bash environment as JSON. for __output in $outputs; do if [[ -z $__done ]]; then - export > ${!__output} - set >> ${!__output} + __dumpEnv > ${!__output} __done=1 else echo -n >> ${!__output} From 5f6375a816803d17c8ec0f7c2ef960c848be6166 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 01:02:47 +0200 Subject: [PATCH 154/555] nix develop: Filter some bash magic variables --- src/nix/develop.cc | 6 +----- src/nix/get-env.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e00f0d575..9b2945ba3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -72,7 +72,7 @@ struct BuildEnvironment void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { - if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) { + if (!ignoreVars.count(name)) { if (auto str = std::get_if(&value)) { out << fmt("%s=%s\n", name, shellEscape(str->value)); if (str->exported) @@ -191,17 +191,13 @@ struct Common : InstallableCommand, MixProfile { std::set ignoreVars{ "BASHOPTS", - "EUID", "HOME", // FIXME: don't ignore in pure mode? - "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", "NIX_REMOTE", "PPID", - "PWD", "SHELLOPTS", - "SHLVL", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 834b84e35..20937b956 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -42,6 +42,20 @@ __dumpEnv() { local type="${BASH_REMATCH[1]}" local __var_name="${BASH_REMATCH[2]}" + if [[ $__var_name =~ ^BASH_ || \ + $__var_name = _ || \ + $__var_name = DIRSTACK || \ + $__var_name = EUID || \ + $__var_name = FUNCNAME || \ + $__var_name = HISTCMD || \ + $__var_name = HOSTNAME || \ + $__var_name = PIPESTATUS || \ + $__var_name = PWD || \ + $__var_name = RANDOM || \ + $__var_name = SHLVL || \ + $__var_name = SECONDS \ + ]]; then continue; fi + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi printf " " From 9fc7da1e08d83378f8b0d66a060d92929c1468cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 01:16:55 +0200 Subject: [PATCH 155/555] Add test for #4992 --- tests/nix-shell.sh | 2 ++ tests/shell.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index aae7cbcde..728313a8a 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -99,3 +99,5 @@ nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo source <(nix print-dev-env -f shell.nix shellDrv) [[ -n $stdenv ]] [[ ${arr1[2]} = "3 4" ]] +[[ ${arr2[1]} = $'\n' ]] +[[ ${arr2[2]} = $'x\ny' ]] diff --git a/tests/shell.nix b/tests/shell.nix index 42d2d2033..1f5526253 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -21,6 +21,7 @@ let pkgs = rec { export PATH=$PATH:$pkg/bin done declare -a arr1=(1 2 "3 4" 5) + declare -a arr2=(x $'\n' $'x\ny') ''; stdenv = mkDerivation { From e50408bd3192156e04571a1546d053f998ab4e2c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 01:17:32 +0200 Subject: [PATCH 156/555] nix develop: Add a test for bash functions --- tests/nix-shell.sh | 1 + tests/shell.nix | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 728313a8a..ec43db349 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -101,3 +101,4 @@ source <(nix print-dev-env -f shell.nix shellDrv) [[ ${arr1[2]} = "3 4" ]] [[ ${arr2[1]} = $'\n' ]] [[ ${arr2[2]} = $'x\ny' ]] +[[ $(fun) = blabla ]] diff --git a/tests/shell.nix b/tests/shell.nix index 1f5526253..70c60276c 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -22,6 +22,9 @@ let pkgs = rec { done declare -a arr1=(1 2 "3 4" 5) declare -a arr2=(x $'\n' $'x\ny') + fun() { + echo blabla + } ''; stdenv = mkDerivation { From 86fb01c4bee0c4f451504c0f6fd2066d2adc8fc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 12:10:48 +0200 Subject: [PATCH 157/555] nix print-dev-env: Add --json flag --- src/nix/develop.cc | 59 +++++++++++++++++++++++++++++++++++----- src/nix/get-env.sh | 1 + src/nix/print-dev-env.md | 37 +++++++++++++++++++++++-- tests/nix-shell.sh | 2 ++ 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9b2945ba3..ca04f297d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -31,6 +31,11 @@ struct BuildEnvironment { bool exported; std::string value; + + bool operator == (const String & other) const + { + return exported == other.exported && value == other.value; + } }; using Array = std::vector; @@ -42,15 +47,13 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; - static BuildEnvironment fromJSON(const Path & path) + static BuildEnvironment fromJSON(std::string_view in) { BuildEnvironment res; std::set exported; - debug("reading environment file '%s'", path); - - auto json = nlohmann::json::parse(readFile(path)); + auto json = nlohmann::json::parse(in); for (auto & [name, info] : json["variables"].items()) { std::string type = info["type"]; @@ -69,6 +72,38 @@ struct BuildEnvironment return res; } + std::string toJSON() const + { + auto res = nlohmann::json::object(); + + auto vars2 = nlohmann::json::object(); + for (auto & [name, value] : vars) { + auto info = nlohmann::json::object(); + if (auto str = std::get_if(&value)) { + info["type"] = str->exported ? "exported" : "var"; + info["value"] = str->value; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "array"; + info["value"] = *arr; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "associative"; + info["value"] = *arr; + } + vars2[name] = std::move(info); + } + res["variables"] = std::move(vars2); + + res["bashFunctions"] = bashFunctions; + + auto json = res.dump(); + + assert(BuildEnvironment::fromJSON(json) == *this); + + return json; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -116,6 +151,11 @@ struct BuildEnvironment else throw Error("bash variable is not a string or array"); } + + bool operator == (const BuildEnvironment & other) const + { + return vars == other.vars && bashFunctions == other.bashFunctions; + } }; const static std::string getEnvSh = @@ -309,7 +349,9 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {BuildEnvironment::fromJSON(strPath), strPath}; + debug("reading environment file '%s'", strPath); + + return {BuildEnvironment::fromJSON(readFile(strPath)), strPath}; } }; @@ -462,7 +504,7 @@ struct CmdDevelop : Common, MixEnvironment } }; -struct CmdPrintDevEnv : Common +struct CmdPrintDevEnv : Common, MixJSON { std::string description() override { @@ -484,7 +526,10 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(store, buildEnvironment); + logger->writeToStdout( + json + ? buildEnvironment.toJSON() + : makeRcScript(store, buildEnvironment)); } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 20937b956..2df3e543f 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -49,6 +49,7 @@ __dumpEnv() { $__var_name = FUNCNAME || \ $__var_name = HISTCMD || \ $__var_name = HOSTNAME || \ + $__var_name = GROUPS || \ $__var_name = PIPESTATUS || \ $__var_name = PWD || \ $__var_name = RANDOM || \ diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index b80252acf..2aad491de 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -8,12 +8,43 @@ R""( # . <(nix print-dev-env nixpkgs#hello) ``` +* Get the build environment in JSON format: + + ```console + # nix print-dev-env nixpkgs#hello --json + ``` + + The output will look like this: + + ```json + { + "bashFunctions": { + "buildPhase": " \n runHook preBuild;\n...", + ... + }, + "variables": { + "src": { + "type": "exported", + "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz" + }, + "postUnpackHooks": { + "type": "array", + "value": ["_updateSourceDateEpochFromSourceRoot"] + }, + ... + } + } + ``` + # Description -This command prints a shell script that can be sourced by `b`ash and -that sets the environment variables and shell functions defined by the -build process of *installable*. This allows you to get a similar build +This command prints a shell script that can be sourced by `bash` and +that sets the variables and shell functions defined by the build +process of *installable*. This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). +With `--json`, the output is a JSON serialisation of the variables and +functions defined by the build process. + )"" diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index ec43db349..f60102f9c 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -96,6 +96,8 @@ echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo # Test 'nix print-dev-env'. +[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] + source <(nix print-dev-env -f shell.nix shellDrv) [[ -n $stdenv ]] [[ ${arr1[2]} = "3 4" ]] From 07790fdddf7705454310633b9e38bd816d23de8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Jun 2021 14:43:06 +0200 Subject: [PATCH 158/555] ref: Add equality operators --- src/libutil/ref.hh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 0be2a7e74..2549ef496 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -73,6 +73,16 @@ public: return ref((std::shared_ptr) p); } + bool operator == (const ref & other) const + { + return p == other.p; + } + + bool operator != (const ref & other) const + { + return p != other.p; + } + private: template From ceda58d112f35a23c7375e0d9b0ebbe53ca93788 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Jun 2021 15:35:14 +0200 Subject: [PATCH 159/555] Formatting --- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6736adb24..96eb4c3d7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -803,14 +803,14 @@ std::map copyPaths(ref srcStore, ref dstStor // Copy the realisation closure processGraph( pool, Realisation::closure(*srcStore, toplevelRealisations), - [&](const Realisation& current) -> std::set { + [&](const Realisation & current) -> std::set { std::set children; - for (const auto& [drvOutput, _] : current.dependentRealisations) { + for (const auto & [drvOutput, _] : current.dependentRealisations) { auto currentChild = srcStore->queryRealisation(drvOutput); if (!currentChild) throw Error( - "Incomplete realisation closure: '%s' is a " - "dependency of '%s' but isn’t registered", + "incomplete realisation closure: '%s' is a " + "dependency of '%s' but isn't registered", drvOutput.to_string(), current.id.to_string()); children.insert(*currentChild); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9657d2adf..813e5c679 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -762,7 +762,7 @@ std::map copyPaths(ref srcStore, ref dstStor CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); std::map copyPaths(ref srcStore, ref dstStore, - const StorePathSet& paths, + const StorePathSet & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From ae0ed53b09cba991551e984f474bf5732d26d3f3 Mon Sep 17 00:00:00 2001 From: Niels Egberts Date: Fri, 9 Jul 2021 21:50:10 +0100 Subject: [PATCH 160/555] toString also coerces a set with an outPath attribute to a string nix-repl> builtins.toString { outPath = "somestring"; } "somestring" --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e8569b654..209a05d11 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3109,7 +3109,7 @@ static RegisterPrimOp primop_toString({ - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`. - - A set containing `{ __toString = self: ...; }`. + - A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`. - An integer. From 2cf14db857fa4be3b60b6497bc4458d5b589154f Mon Sep 17 00:00:00 2001 From: Niels Egberts Date: Sun, 11 Jul 2021 11:15:06 +0100 Subject: [PATCH 161/555] Throw on unexpected input for --delete-older-than '--delete-older-than 10' deletes the generations older than a single day, and '--delete-older-than 12m' deletes all generations older than 12 days. This changes makes it throw on those invalid inputs, and gives an example of a valid input. --- src/libstore/profiles.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 5d1723886..75db8f9df 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -211,6 +211,9 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) { + if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') + throw Error("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); + time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); auto days = string2Int(strDays); From 91d2e8d5adf8d4ac941faef5cb3a020f07238db9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Jul 2021 15:04:46 +0200 Subject: [PATCH 162/555] Error -> UsageError --- src/libstore/profiles.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 75db8f9df..84a21c0ba 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -212,14 +212,14 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) { if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') - throw Error("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); + throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); auto days = string2Int(strDays); if (!days || *days < 1) - throw Error("invalid number of days specifier '%1%'", timeSpec); + throw UsageError("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - *days * 24 * 3600; From 02dff9e5299eccd8199dfbf1b5c20bf6ed9f07bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Jul 2021 17:17:13 +0200 Subject: [PATCH 163/555] Style --- src/libstore/parsed-derivations.cc | 28 +++++++++++++--------------- src/libstore/parsed-derivations.hh | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 345235d66..caddba9b1 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -126,6 +126,7 @@ bool ParsedDerivation::substitutesAllowed() const } static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -135,9 +136,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; - for (auto & i : drv.outputs) { + for (auto & i : drv.outputs) outputs[i.first] = hashPlaceholder(i.first); - } json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -165,7 +165,7 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s namely, strings, integers, nulls, Booleans, and arrays and objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ -std::string writeStructuredAttrsShell(nlohmann::json & json) +std::string writeStructuredAttrsShell(const nlohmann::json & json) { auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { @@ -189,42 +189,40 @@ std::string writeStructuredAttrsShell(nlohmann::json & json) std::string jsonSh; - for (auto i = json.begin(); i != json.end(); ++i) { + for (auto & [key, value] : json.items()) { - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); + if (!std::regex_match(key, shVarName)) continue; auto s = handleSimpleType(value); if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); + jsonSh += fmt("declare %s=%s\n", key, *s); else if (value.is_array()) { std::string s2; bool good = true; - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); + for (auto & value2 : value) { + auto s3 = handleSimpleType(value2); if (!s3) { good = false; break; } s2 += *s3; s2 += ' '; } if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + jsonSh += fmt("declare -a %s=(%s)\n", key, s2); } else if (value.is_object()) { std::string s2; bool good = true; - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); + for (auto & [key2, value2] : value.items()) { + auto s3 = handleSimpleType(value2); if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + s2 += fmt("[%s]=%s ", shellEscape(key2), *s3); } if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + jsonSh += fmt("declare -A %s=(%s)\n", key, s2); } } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 5e3fb93ca..effcf099d 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -40,6 +40,6 @@ public: std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; -std::string writeStructuredAttrsShell(nlohmann::json & json); +std::string writeStructuredAttrsShell(const nlohmann::json & json); } From 7bc17a903bf3b71ce867ac5316b7e0a9e03c68b2 Mon Sep 17 00:00:00 2001 From: illustris Date: Tue, 13 Jul 2021 15:34:14 +0530 Subject: [PATCH 164/555] fixed output derivations: fix incorrect responses for getpwuid Passing nscd socket into the build environment causes unexpected behavior in programs that make getpwuid and other related calls. relevant threads: - https://github.com/NixOS/nix/issues/4991 - https://discourse.nixos.org/t/haunted-nix-build-breaks-isolation/13869 --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8320dd1c4..d4a412ff6 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1754,7 +1754,7 @@ void LocalDerivationGoal::runChild() /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations within a pure derivation. */ - for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); } From 037c86ee04d681170cedaf71f9d75c186abe9719 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 13 Jul 2021 11:44:19 +0200 Subject: [PATCH 165/555] nix develop: Search in `devShells.${system}` by default Make `nix develop .#foo` search `.#devShells.${system}.foo` first --- src/nix/develop.cc | 6 ++++++ src/nix/develop.md | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 26f53db09..9ac2791f8 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -326,6 +326,12 @@ struct Common : InstallableCommand, MixProfile { return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; } + Strings getDefaultFlakeAttrPathPrefixes() override + { + auto res = SourceExprCommand::getDefaultFlakeAttrPathPrefixes(); + res.emplace_front("devShells." + settings.thisSystem.get()); + return res; + } StorePath getShellOutPath(ref store) { diff --git a/src/nix/develop.md b/src/nix/develop.md index e71d9f8aa..c86c4872b 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -84,11 +84,20 @@ the flake's `nixConfig` attribute. # Flake output attributes -If no flake output attribute is given, `nix run` tries the following +If no flake output attribute is given, `nix develop` tries the following flake output attributes: * `devShell.` * `defaultPackage.` +If a flake output *name* is given, `nix develop` tries the following flake +output attributes: + +* `devShells..` + +* `packages..` + +* `legacyPackages..` + )"" From 43d5c5f87b0f1cc9405709a777d66496b4ca4e21 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 13 Jul 2021 13:44:47 +0200 Subject: [PATCH 166/555] Make `nix flake show` display the `devShells` --- src/nix/flake.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9055b07eb..ce82eb8e8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -914,6 +914,7 @@ struct CmdFlakeShow : FlakeCommand logger->cout("%s: %s '%s'", headerPrefix, attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : + attrPath.size() >= 2 && attrPath[0] == "devShells" ? "development environment" : attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : "package", @@ -932,6 +933,7 @@ struct CmdFlakeShow : FlakeCommand || ((attrPath.size() == 1 || attrPath.size() == 2) && (attrPath[0] == "checks" || attrPath[0] == "packages" + || attrPath[0] == "devShells" || attrPath[0] == "apps")) ) { @@ -940,7 +942,7 @@ struct CmdFlakeShow : FlakeCommand else if ( (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) - || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages")) + || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) ) { if (visitor.isDerivation()) From 797e260e3a322937bd31f94d166beafc283f6ed7 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 13 Jul 2021 17:23:29 +0200 Subject: [PATCH 167/555] Make `nix flake check` aware of `devShells` --- src/nix/flake-check.md | 1 + src/nix/flake.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index 8ef932954..d995d6274 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -33,6 +33,7 @@ The following flake output attributes must be derivations: * `checks.`*system*`.`*name* * `defaultPackage.`*system*` * `devShell.`*system*` +* `devShells.`*system*`.`*name*` * `nixosConfigurations.`*name*`.config.system.build.toplevel * `packages.`*system*`.`*name* diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce82eb8e8..23feed24b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -486,7 +486,7 @@ struct CmdFlakeCheck : FlakeCommand } } - else if (name == "packages") { + else if (name == "packages" || name == "devShells") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); From 99f8fc995b2f080cc0a6fe934c8d9c777edc3751 Mon Sep 17 00:00:00 2001 From: Pamplemousse Date: Tue, 13 Jul 2021 14:23:24 -0700 Subject: [PATCH 168/555] libexpr: Fix read out-of-bound on the heap Signed-off-by: Pamplemousse --- src/libexpr/lexer.l | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 7298419d9..27975dc9e 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -38,11 +38,13 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) loc->first_line = loc->last_line; loc->first_column = loc->last_column; - while (len--) { + for (size_t i = 0; i < len; i++) { switch (*s++) { case '\r': - if (*s == '\n') /* cr/lf */ + if (*s == '\n') { /* cr/lf */ + i++; s++; + } /* fall through */ case '\n': ++loc->last_line; From 307977963ca953ad11003bcd4c95f7d2308012f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Jul 2021 12:25:53 +0200 Subject: [PATCH 169/555] nlohmann_json: Update to 3.9.1, fix use of internal copy --- Makefile | 2 +- flake.nix | 2 - src/nlohmann/json.hpp | 16229 +++++++++++++++++++++++------------- src/nlohmann/json_fwd.hpp | 78 + 4 files changed, 10714 insertions(+), 5597 deletions(-) create mode 100644 src/nlohmann/json_fwd.hpp diff --git a/Makefile b/Makefile index dd259e5cd..c7d8967c8 100644 --- a/Makefile +++ b/Makefile @@ -33,4 +33,4 @@ endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src diff --git a/flake.nix b/flake.nix index 5430d3904..ee76d9188 100644 --- a/flake.nix +++ b/flake.nix @@ -89,7 +89,6 @@ openssl sqlite libarchive boost - nlohmann_json lowdown gmock ] @@ -334,7 +333,6 @@ xz pkgs.perl boost - nlohmann_json ] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index c9af0bed3..a70aaf8cb 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,12 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 +| | |__ | | | | | | version 3.9.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . +Copyright (c) 2013-2019 Niels Lohmann . Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,770 +27,44 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ #define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 +#define NLOHMANN_JSON_VERSION_MINOR 9 +#define NLOHMANN_JSON_VERSION_PATCH 1 #include // all_of, find, for_each -#include // assert -#include // and, not, or #include // nullptr_t, ptrdiff_t, size_t #include // hash, less #include // initializer_list #include // istream, ostream #include // random_access_iterator_tag +#include // unique_ptr #include // accumulate #include // string, stoi, to_string #include // declval, forward, move, pair, swap - -// #include -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string #include // vector -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -/*! -@brief default JSONSerializer template argument +// #include -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer> -class basic_json; +#include -/*! -@brief JSON Pointer +// #include -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - -@since version 2.0.0 -*/ -template -class json_pointer; - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} // namespace nlohmann - -#endif - -// #include - - -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them - -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" -#endif - -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) -#else - #define JSON_DEPRECATED -#endif - -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) -#else - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) -#endif - -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER -#endif - -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif - -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif - -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } - -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. - -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> - -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json - -// #include - - -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type - -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; - -template -using uncvref_t = typename std::remove_cv::type>::type; - -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; - -template -struct merge_and_renumber; - -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; - -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; - -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; - -template -using index_sequence_for = make_index_sequence; - -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; - -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; - -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval - -// #include - -// #include - - -#include // random_access_iterator_tag - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; - -template -struct iterator_traits::value>> -{ - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; -}; -} -} - -// #include - -// #include - - -#include - -// #include - - -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ -namespace detail -{ -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - void operator=(nonesuch const&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template