diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 5e8f5d929..383717bfc 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -3,7 +3,7 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. - _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos). +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org). ## Suggestions for PRs diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..ec7ab4516 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,26 @@ +name: Backport +on: + pull_request_target: + types: [closed, labeled] +jobs: + backport: + name: Backport Pull Request + if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + # required to find all branches + fetch-depth: 0 + - name: Create backport PRs + # should be kept in sync with `version` + uses: zeebe-io/backport-action@v0.0.7 + with: + # Config README: https://github.com/zeebe-io/backport-action#backport-action + github_token: ${{ secrets.GITHUB_TOKEN }} + github_workspace: ${{ github.workspace }} + pull_description: |- + Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + # should be kept in sync with `uses` + version: v0.0.5 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17a79dc97..1b655e27d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,27 +1,32 @@ name: "Test" + on: pull_request: push: + jobs: + tests: + needs: [check_cachix] strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - + timeout-minutes: 60 steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v13 + - uses: cachix/install-nix-action@v16 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 + if: needs.check_cachix.outputs.secret == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - #- run: nix flake check - - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) + - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)') + check_cachix: name: Cachix secret present for installer tests runs-on: ubuntu-latest @@ -33,6 +38,7 @@ jobs: env: _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + installer: needs: [tests, check_cachix] if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' @@ -40,11 +46,11 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v13 + - uses: cachix/install-nix-action@v16 - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' @@ -52,6 +58,7 @@ jobs: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - id: prepare-installer run: scripts/prepare-installer-for-github-actions + installer_test: needs: [installer, check_cachix] if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' @@ -60,9 +67,9 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.4.0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v13 + - uses: cachix/install-nix-action@v16 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" diff --git a/.gitignore b/.gitignore index 37aada307..2889a56eb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ perl/Makefile.config /doc/manual/*.1 /doc/manual/*.5 /doc/manual/*.8 +/doc/manual/generated/* /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json @@ -25,8 +26,6 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-reduce-build -/scripts/nix-http-export.cgi /scripts/nix-profile-daemon.sh # /src/libexpr/ @@ -39,6 +38,7 @@ perl/Makefile.config # /src/libstore/ *.gen.* +/src/libstore/tests/libstore-tests # /src/libutil/ /src/libutil/tests/libutil-tests @@ -56,9 +56,6 @@ perl/Makefile.config /src/nix-prefetch-url/nix-prefetch-url -# /src/nix-daemon/ -/src/nix-daemon/nix-daemon - /src/nix-collect-garbage/nix-collect-garbage # /src/nix-channel/ @@ -76,12 +73,12 @@ perl/Makefile.config # /tests/ /tests/test-tmp /tests/common.sh -/tests/dummy /tests/result* /tests/restricted-innocent /tests/shell /tests/shell.drv /tests/config.nix +/tests/ca/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/.version b/.version index 7208c2182..68151b2e1 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.4 \ No newline at end of file +2.5 \ No newline at end of file diff --git a/Makefile b/Makefile index b0636cf49..02228910a 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ makefiles = \ src/libutil/local.mk \ src/libutil/tests/local.mk \ src/libstore/local.mk \ + src/libstore/tests/local.mk \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ @@ -12,6 +13,8 @@ makefiles = \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ misc/bash/local.mk \ + misc/fish/local.mk \ + misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ misc/upstart/local.mk \ @@ -32,4 +35,4 @@ endif include mk/lib.mk # GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -fstack-usage -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src diff --git a/Makefile.config.in b/Makefile.config.in index fd92365eb..c8c4446b4 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ BOOST_LDFLAGS = @BOOST_LDFLAGS@ diff --git a/README.md b/README.md index 4686010ef..80d6f128c 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ build nix from source with nix-build or how to get a development environment. - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) -- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) +- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff new file mode 100644 index 000000000..e659bf470 --- /dev/null +++ b/boehmgc-coroutine-sp-fallback.diff @@ -0,0 +1,45 @@ +diff --git a/pthread_stop_world.c b/pthread_stop_world.c +index 4b2c429..1fb4c52 100644 +--- a/pthread_stop_world.c ++++ b/pthread_stop_world.c +@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) + struct GC_traced_stack_sect_s *traced_stack_sect; + pthread_t self = pthread_self(); + word total_size = 0; ++ size_t stack_limit; ++ pthread_attr_t pattr; + + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); +@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + hi = p->altstack + p->altstack_size; + /* FIXME: Need to scan the normal stack too, but how ? */ + /* FIXME: Assume stack grows down */ ++ } else { ++ if (pthread_getattr_np(p->id, &pattr)) { ++ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); ++ } ++ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { ++ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); ++ } ++ if (pthread_attr_destroy(&pattr)) { ++ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); ++ } ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ #ifndef STACK_GROWS_UP ++ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack ++ lo = hi - stack_limit; ++ } ++ #else ++ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." ++ #endif + } + GC_push_all_stack_sections(lo, hi, traced_stack_sect); + # ifdef STACK_GROWS_UP diff --git a/config/config.guess b/config/config.guess index 699b3a10b..1972fda8e 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-11-19' +timestamp='2021-01-25' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -188,10 +188,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ - "/sbin/$sysctl" 2>/dev/null || \ - "/usr/sbin/$sysctl" 2>/dev/null || \ + /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ echo unknown)) case "$UNAME_MACHINE_ARCH" in aarch64eb) machine=aarch64_be-unknown ;; @@ -996,6 +995,9 @@ EOF k1om:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; m32r*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; @@ -1084,7 +1086,7 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-"$LIBC" exit ;; - riscv32:Linux:*:* | riscv64:Linux:*:*) + riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) @@ -1480,8 +1482,8 @@ EOF i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos exit ;; - i*86:AROS:*:*) - echo "$UNAME_MACHINE"-pc-aros + *:AROS:*:*) + echo "$UNAME_MACHINE"-unknown-aros exit ;; x86_64:VMkernel:*:*) echo "$UNAME_MACHINE"-unknown-esx diff --git a/config/config.sub b/config/config.sub index 19c9553b1..63c1f1c8b 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-12-02' +timestamp='2021-01-08' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -1185,6 +1185,7 @@ case $cpu-$vendor in | k1om \ | le32 | le64 \ | lm32 \ + | loongarch32 | loongarch64 | loongarchx32 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ @@ -1229,7 +1230,7 @@ case $cpu-$vendor in | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ | pru \ | pyramid \ - | riscv | riscv32 | riscv64 \ + | riscv | riscv32 | riscv32be | riscv64 | riscv64be \ | rl78 | romp | rs6000 | rx \ | s390 | s390x \ | score \ @@ -1682,11 +1683,14 @@ fi # Now, validate our (potentially fixed-up) OS. case $os in - # Sometimes we do "kernel-abi", so those need to count as OSes. + # Sometimes we do "kernel-libc", so those need to count as OSes. musl* | newlib* | uclibc*) ;; - # Likewise for "kernel-libc" - eabi | eabihf | gnueabi | gnueabihf) + # Likewise for "kernel-abi" + eabi* | gnueabi*) + ;; + # VxWorks passes extra cpu info in the 4th filed. + simlinux | simwindows | spe) ;; # Now accept the basic system types. # The portable systems comes first. @@ -1750,6 +1754,8 @@ case $kernel-$os in ;; kfreebsd*-gnu* | kopensolaris*-gnu*) ;; + vxworks-simlinux | vxworks-simwindows | vxworks-spe) + ;; nto-qnx*) ;; os2-emx) diff --git a/configure.ac b/configure.ac index 6c36787f3..65478ecc5 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_INIT([nix],[m4_esyscmd(bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX")]) AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -9,8 +9,7 @@ AC_PROG_SED AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) -AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], - [Platform identifier (e.g., `i686-linux').]), +AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (e.g., `i686-linux').]), [system=$withval], [case "$host_cpu" in i*86) @@ -33,14 +32,6 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";; esac]) -sys_name=$(uname -s | tr 'A-Z ' 'a-z_') - -case $sys_name in - cygwin*) - sys_name=cygwin - ;; -esac - AC_MSG_RESULT($system) AC_SUBST(system) AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) @@ -64,10 +55,12 @@ AC_SYS_LARGEFILE # Solaris-specific stuff. AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then +case "$host_os" in + solaris*) # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi + LDFLAGS="-lsocket -lnsl $LDFLAGS" + ;; +esac # Check for pubsetbuf. @@ -127,8 +120,7 @@ NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) -AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], - [path of the Nix store (defaults to /nix/store)]), +AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix store (defaults to /nix/store)]), storedir=$withval, storedir='/nix/store') AC_SUBST(storedir) @@ -152,13 +144,12 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="-latomic $LDFLAGS" fi PKG_PROG_PKG_CONFIG -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), +AC_ARG_ENABLE(shared, AS_HELP_STRING([--enable-shared],[Build shared libraries for Nix [default=yes]]), shared=$enableval, shared=yes) if test "$shared" = yes; then AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) @@ -213,30 +204,32 @@ AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) # Look for libseccomp, required for Linux sandboxing. -if test "$sys_name" = linux; then - AC_ARG_ENABLE([seccomp-sandboxing], - AC_HELP_STRING([--disable-seccomp-sandboxing], - [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] - )) - if test "x$enable_seccomp_sandboxing" != "xno"; then - PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], - [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) - have_seccomp=1 - AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) - else +case "$host_os" in + linux*) + AC_ARG_ENABLE([seccomp-sandboxing], + AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!) + ])) + if test "x$enable_seccomp_sandboxing" != "xno"; then + PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + have_seccomp=1 + AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + else + have_seccomp= + fi + ;; + *) have_seccomp= - fi -else - have_seccomp= -fi + ;; +esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) AC_CHECK_HEADERS([aws/s3/S3Client.h], - [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) - enable_s3=1], [enable_s3=]) + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], + [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=]) AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) @@ -249,8 +242,7 @@ fi # Whether to use the Boehm garbage collector. -AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], - [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), +AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), gc=$enableval, gc=yes) if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) @@ -264,11 +256,12 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # documentation generation switch -AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], - [disable documentation generation]), +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) +# Look for lowdown library. +PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.8.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) @@ -280,13 +273,14 @@ AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) # This is needed if bzip2 is a static library, and the Nix libraries # are dynamic. -if test "$(uname)" = "Darwin"; then +case "${host_os}" in + darwin*) LDFLAGS="-all_load $LDFLAGS" -fi + ;; +esac -AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], - [path of a statically-linked shell to use as /bin/sh in sandboxes]), +AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) @@ -301,6 +295,6 @@ done rm -f Makefile.config -AC_CONFIG_HEADER([config.h]) +AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([]) AC_OUTPUT diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 416a7fdba..92c7b1a31 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -6,9 +6,11 @@ builtins: concatStrings (map (name: let builtin = builtins.${name}; in - " - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args) - + " \n\n" - + concatStrings (map (s: " ${s}\n") (splitLines builtin.doc)) + "\n\n" + "
${name} " + + concatStringsSep " " (map (s: "${s}") builtin.args) + + "
" + + "
\n\n" + + builtin.doc + + "\n\n
" ) (attrNames builtins)) - diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 964b57086..244cfa0c2 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,4 +1,4 @@ -command: +{ command, renderLinks ? false }: with builtins; with import ./utils.nix; @@ -20,7 +20,11 @@ let categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands))); listCommands = cmds: concatStrings (map (name: - "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n") + "* " + + (if renderLinks + then "[`${command} ${name}`](./${appendName filename name}.md)" + else "`${command} ${name}`") + + " - ${cmds.${name}.description}\n") (attrNames cmds)); in "where *subcommand* is one of the following:\n\n" @@ -89,7 +93,7 @@ let in let - manpages = processCommand { filename = "nix"; command = "nix"; def = command; }; + manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; }; summary = concatStrings (map (manpage: " - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages); in (listToAttrs manpages) // { "SUMMARY.md" = summary; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 271529b38..e43d9f2fb 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,7 +1,5 @@ ifeq ($(doc_generate),yes) -MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md) - # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ @@ -46,7 +44,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' + $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }' $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @@ -64,6 +62,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -74,17 +73,28 @@ $(d)/builtins.json: $(bindir)/nix install: $(docdir)/manual/index.html # Generate 'nix' manpages. -install: $(d)/src/command-ref/new-cli +install: $(mandir)/man1/nix3-manpages +man: doc/manual/generated/man1/nix3-manpages +all: doc/manual/generated/man1/nix3-manpages + +$(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages + @mkdir -p $(DESTDIR)$$(dirname $@) + $(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@) + +doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli + @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ name=$$(basename $$i .md); \ + tmpFile=$$(mktemp); \ if [[ $$name = SUMMARY ]]; then continue; fi; \ - printf "Title: %s\n\n" "$$name" > $$i.tmp; \ - cat $$i >> $$i.tmp; \ - lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \ + printf "Title: %s\n\n" "$$name" > $$tmpFile; \ + cat $$i >> $$tmpFile; \ + lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ + rm $$tmpFile; \ done + @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual - @cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) + $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 448fee803..8d9b061ba 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -9,6 +9,7 @@ - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) + - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - [Single-User Mode](installation/single-user.md) - [Multi-User Mode](installation/multi-user.md) @@ -70,6 +71,8 @@ - [Hacking](contributing/hacking.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) + - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 3140170ab..44b7ba86d 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -16,8 +16,9 @@ By default Nix reads settings from the following places: will be loaded in reverse order. Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If these are unset, it will look in - `$HOME/.config/nix.conf`. + and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to + `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` + as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b709ca9d1..6e2403461 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables: A colon-separated list of directories used to look up Nix expressions enclosed in angle brackets (i.e., ``). For instance, the value - + /home/eelco/Dev:/etc/nixos - + will cause Nix to look for paths relative to `/home/eelco/Dev` and `/etc/nixos`, in this order. It is also possible to match paths against a prefix. For example, the value - + nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos - + will cause Nix to search for `` in `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - + If a path in the Nix search path starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. The tarball must consist of a single top-level directory. For example, setting `NIX_PATH` to - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz - - tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09 - channel. - - A following shorthand can be used to refer to the official channels: - - nixpkgs=channel:nixos-15.09 - - The search path can be extended using the `-I` option, which takes - precedence over `NIX_PATH`. + + nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + + tells Nix to download and use the current contents of the + `master` branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official nixos.org channels (see + [the manual for `nix-channel`](nix-channel.md)) can be abbreviated + as `channel:`. For instance, the following two + values of `NIX_PATH` are equivalent: + + nixpkgs=channel:nixos-21.05 + nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + + The Nix search path can also be extended using the `-I` option to + many Nix commands, which takes precedence over `NIX_PATH`. - `NIX_IGNORE_SYMLINK_STORE`\ Normally, the Nix store directory (typically `/nix/store`) is not @@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables: builds are deployed to machines where `/nix/store` resolves differently. If you are sure that you’re not going to do that, you can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - + Note that if you’re symlinking the Nix store so that you can put it on another file system than the root file system, on Linux you’re better off using `bind` mount points, e.g., @@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables: $ mkdir /nix $ mount -o bind /mnt/otherdisk/nix /nix ``` - + Consult the mount 8 manual page for details. - `NIX_STORE_DIR`\ diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 9138fa05a..8d6abaf52 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -238,7 +238,16 @@ a number of possible ways: ## Examples -To install a specific version of `gcc` from the active Nix expression: +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env -iA gcc40mips +installing `gcc-4.0.2' +$ nix-env -iA xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: ```console $ nix-env --install gcc-3.3.2 @@ -246,6 +255,9 @@ installing `gcc-3.3.2' uninstalling `gcc-3.1' ``` +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + Note the previously installed version is removed, since `--preserve-installed` was not specified. @@ -256,13 +268,6 @@ $ nix-env --install gcc installing `gcc-3.3.2' ``` -To install using a specific attribute: - -```console -$ nix-env -i -A gcc40mips -$ nix-env -i -A xorg.xorgserver -``` - To install all derivations in the Nix expression `foo.nix`: ```console @@ -374,22 +379,29 @@ For the other flags, see `--install`. ## Examples ```console -$ nix-env --upgrade gcc +$ nix-env --upgrade -A nixpkgs.gcc upgrading `gcc-3.3.1' to `gcc-3.4' ``` +When there are no updates available, nothing will happen: + ```console -$ nix-env -u gcc-3.3.2 --always (switch to a specific version) +$ nix-env --upgrade -A nixpkgs.pan +``` + +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + +```console +$ nix-env -u gcc-3.3.2 --always upgrading `gcc-3.4' to `gcc-3.3.2' ``` -```console -$ nix-env --upgrade pan -(no upgrades available, so nothing happens) -``` +To try to upgrade everything +(matching packages based on the part of the derivation name without version): ```console -$ nix-env -u (try to upgrade everything) +$ nix-env -u upgrading `hello-2.1.2' to `hello-2.1.3' upgrading `mozilla-1.2' to `mozilla-1.4' ``` @@ -401,7 +413,7 @@ of a derivation `x` by looking at their respective `name` attributes. The names (e.g., `gcc-3.3.1` are split into two parts: the package name (`gcc`), and the version (`3.3.1`). The version part starts after the first dash not followed by a letter. `x` is considered an upgrade of `y` -if their package names match, and the version of `y` is higher that that +if their package names match, and the version of `y` is higher than that of `x`. The versions are compared by splitting them into contiguous components diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index dcd7cc70c..873311649 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -11,8 +11,8 @@ [`--command` *cmd*] [`--run` *cmd*] [`--exclude` *regexp*] - [--pure] - [--keep *name*] + [`--pure`] + [`--keep` *name*] {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]} # Description @@ -78,9 +78,7 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. Note that (depending on your Bash - installation) `/etc/bashrc` is still sourced, so any variables set - there will affect the interactive shell. + retained. - `--packages` / `-p` *packages*…\ Set up an environment in which the specified packages are present. @@ -112,13 +110,19 @@ shell in which to build it: ```console $ nix-shell '' -A pan -[nix-shell]$ unpackPhase +[nix-shell]$ eval ${unpackPhase:-unpackPhase} [nix-shell]$ cd pan-* -[nix-shell]$ configurePhase -[nix-shell]$ buildPhase +[nix-shell]$ eval ${configurePhase:-configurePhase} +[nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan ``` +The reason we use form `eval ${configurePhase:-configurePhase}` here is because +those packages that override these phases do so by exporting the overridden +values in the environment variable of the same name. +Here bash is being told to either evaluate the contents of 'configurePhase', +if it exists as a variable, otherwise evaluate the configurePhase function. + To clear the environment first, and do some additional automatic initialisation of the interactive shell: diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 7a131dc02..26292f1bb 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -125,7 +125,7 @@ Special exit codes: - `104`\ Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducable. + resulting output is not binary reproducible. With the `--keep-going` flag it's possible for multiple failures to occur, in this case the 1xx status codes are or combined using binary diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 47862bc09..7ee1a26bc 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options: }: ... ``` - So if you call this Nix expression (e.g., when you do `nix-env -i + So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](../expressions/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., - `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that + `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 0132867c8..01a1b1e73 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -3,7 +3,7 @@ ## Goals Purpose of this document is to provide a clear direction to **help design -delightful command line** experience. This document contain guidelines to +delightful command line** experience. This document contains guidelines to follow to ensure a consistent and approachable user experience. ## Overview @@ -103,7 +103,7 @@ impacted the most by bad user experience. # Help is essential Help should be built into your command line so that new users can gradually -discover new features when they need them. +discover new features when they need them. ## Looking for help @@ -115,7 +115,7 @@ The rules are: - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or `nix help`). -- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show +- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show a summary** of most common use cases. Summary is presented on the STDOUT without any use of PAGER. - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the @@ -176,7 +176,7 @@ $ nix init --template=template#pyton ------------------------------------------------------------------------ Initializing Nix project at `/path/to/here`. Select a template for you new project: - |> template#pyton + |> template#python template#python-pip template#python-poetry ``` @@ -230,17 +230,17 @@ Now **Learn** part of the output is where you educate users. You should only show it when you know that a build will take some time and not annoy users of the builds that take only few seconds. -Every feature like this should go though a intensive review and testing to -collect as much a feedback as possible and to fine tune every little detail. If +Every feature like this should go through an intensive review and testing to +collect as much feedback as possible and to fine tune every little detail. If done right this can be an awesome features beginners and advance users will love, but if not done perfectly it will annoy users and leave bad impression. # Input -Input to a command is provided via `ARGUMENTS` and `OPTIONS`. +Input to a command is provided via `ARGUMENTS` and `OPTIONS`. `ARGUMENTS` represent a required input for a function. When choosing to use -`ARGUMENT` over function please be aware of the downsides that come with it: +`ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it: - User will need to remember the order of `ARGUMENTS`. This is not a problem if there is only one `ARGUMENT`. @@ -253,7 +253,7 @@ developer consider the downsides and choose wisely. ## Naming the `OPTIONS` -Then only naming convention - apart from the ones mentioned in Naming the +The only naming convention - apart from the ones mentioned in Naming the `COMMANDS` section is how flags are named. Flags are a type of `OPTION` that represent an option that can be turned ON of @@ -271,12 +271,12 @@ to improve the discoverability of possible input. A new user will most likely not know which `ARGUMENTS` and `OPTIONS` are required or which values are possible for those options. -In cases, the user might not provide the input or they provide wrong input, -rather then show the error, prompt a user with an option to find and select +In case the user does not provide the input or they provide wrong input, +rather than show the error, prompt a user with an option to find and select correct input (see examples). Prompting is of course not required when TTY is not attached to STDIN. This -would mean that scripts wont need to handle prompt, but rather handle errors. +would mean that scripts won't need to handle prompt, but rather handle errors. A place to use prompt and provide user with interactive select @@ -300,9 +300,9 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org # Output -Terminal output can be quite limiting in many ways. Which should forces us to +Terminal output can be quite limiting in many ways. Which should force us to think about the experience even more. As with every design the output is a compromise between being terse and being verbose, between showing help to beginners and annoying advance users. For this it is important that we know what are the priorities. Nix command line should be first and foremost written with beginners in mind. -But users wont stay beginners for long and what was once useful might quickly +But users won't stay beginners for long and what was once useful might quickly become annoying. There is no golden rule that we can give in this guideline that would make it easier how to draw a line and find best compromise. @@ -342,7 +342,7 @@ also allowing them to redirect content to a file. For example: ```shell $ nix build > build.txt ------------------------------------------------------------------------ - Error! Atrribute `bin` missing at (1:94) from string. + Error! Attribute `bin` missing at (1:94) from string. ------------------------------------------------------------------------ 1| with import { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } "" @@ -408,7 +408,7 @@ Above command clearly states that command successfully completed. And in case of `nix build`, which is a command that might take some time to complete, it is equally important to also show that a command started. -## Text alignment +## Text alignment Text alignment is the number one design element that will present all of the Nix commands as a family and not as separate tools glued together. @@ -419,7 +419,7 @@ The format we should follow is: $ nix COMMAND VERB_1 NOUN and other words VERB__1 NOUN and other words - |> Some details + |> Some details ``` Few rules that we can extract from above example: @@ -444,13 +444,13 @@ is not even notable, therefore relying on it wouldn’t make much sense. **The bright text is much better supported** across terminals and color schemes. Most of the time the difference is perceived as if the bright text -would be bold. +would be bold. ## Colors Humans are already conditioned by society to attach certain meaning to certain colors. While the meaning is not universal, a simple collection of colors is -used to represent basic emotions. +used to represent basic emotions. Colors that can be used in output @@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool. ### Progress -For longer running commands we should provide and overview of the progress. +For longer running commands we should provide and overview the progress. This is shown best in `nix build` example: ```shell @@ -553,9 +553,9 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -566,7 +566,7 @@ $ nix build --option substitutors https://cache.example.org There are many ways that you can control verbosity. -Verbosity levels are: +Verbosity levels are: - `ERROR` (level 0) - `WARN` (level 1) @@ -586,4 +586,4 @@ There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and # Appendix 1: Commands naming exceptions -`nix init` and `nix repl` are well established +`nix init` and `nix repl` are well established diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md index 5b208df67..000595815 100644 --- a/doc/manual/src/expressions/advanced-attributes.md +++ b/doc/manual/src/expressions/advanced-attributes.md @@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes. - `preferLocalBuild`\ If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivaton will be built locally instead of forwarded + possible, the derivation will be built locally instead of forwarded to a remote machine. This is appropriate for trivial builders where the cost of doing a download or remote build would exceed the cost of building locally. diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md index c16b2805f..c631a8453 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/expressions/builtins-prefix.md @@ -9,7 +9,8 @@ scope. Instead, you can access them through the `builtins` built-in value, which is a set that contains all built-in functions and values. For instance, `derivation` is also available as `builtins.derivation`. - - `derivation` *attrs*; `builtins.derivation` *attrs*\ - - `derivation` is described in [its own section](derivations.md). - +
+
derivation attrs; + builtins.derivation attrs
+

derivation is described in + its own section.

diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/expressions/builtins-suffix.md new file mode 100644 index 000000000..a74db2857 --- /dev/null +++ b/doc/manual/src/expressions/builtins-suffix.md @@ -0,0 +1 @@ +
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md index 2a1306e32..6b93e692c 100644 --- a/doc/manual/src/expressions/expression-syntax.md +++ b/doc/manual/src/expressions/expression-syntax.md @@ -26,7 +26,7 @@ elements (referenced from the figure by number): called with three arguments: `stdenv`, `fetchurl`, and `perl`. They are needed to build Hello, but we don't know how to build them here; that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages packages; it provides a + is used by almost all Nix Packages; it provides a “standard” environment consisting of the things you would expect in a basic Unix environment: a C/C++ compiler (GCC, to be precise), the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index b7fd6f4c6..268b44f4c 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -17,12 +17,12 @@ order of precedence (from strongest to weakest binding). | String Concatenation | *string1* `+` *string2* | left | String concatenation. | 7 | | Not | `!` *e* | none | Boolean negation. | 8 | | Update | *e1* `//` *e2* | right | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes). | 9 | -| Less Than | *e1* `<` *e2*, | none | Arithmetic comparison. | 10 | -| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than | *e1* `>` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic comparison. | 10 | +| Less Than | *e1* `<` *e2*, | none | Arithmetic/lexicographic comparison. | 10 | +| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than | *e1* `>` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | | Equality | *e1* `==` *e2* | none | Equality. | 11 | | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | -| Logical OR | *e1* `\|\|` *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| e2`). | 14 | +| Logical OR | *e1* || *e2* | left | Logical OR. | 13 | +| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to !e1 || e2). | 14 | diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index ce31029cc..75ae9f2eb 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -64,7 +64,7 @@ Nix has the following basic data types: the start of each line. To be precise, it strips from each line a number of spaces equal to the minimal indentation of the string as a whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two space, while the third + the first and second line are indented two spaces, while the third line is indented four spaces. Thus, two spaces are stripped from each line, so the resulting string is @@ -139,6 +139,13 @@ Nix has the following basic data types: environment variable `NIX_PATH` will be searched for the given file or directory name. + Antiquotation is supported in any paths except those in angle brackets. + `./${foo}-${bar}.nix` is a more convenient way of writing + `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At + least one slash must appear *before* any antiquotations for this to be + recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division + operation. `./a.${foo}/b.${bar}` is a path. + - *Booleans* with values `true` and `false`. - The null value, denoted as `null`. diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md index 6f730a936..7f0d8f841 100644 --- a/doc/manual/src/expressions/simple-building-testing.md +++ b/doc/manual/src/expressions/simple-building-testing.md @@ -1,6 +1,6 @@ # Building and Testing -You can now try to build Hello. Of course, you could do `nix-env -i +You can now try to build Hello. Of course, you could do `nix-env -f . -iA hello`, but you may not want to install a possibly broken package just yet. The best way to test the package is by using the command `nix-build`, which builds a Nix expression and creates a symlink named diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index d21a51a82..ed1efffd8 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -1,9 +1,9 @@ # Building Nix from Source -After unpacking or checking out the Nix sources, issue the following -commands: +After cloning Nix's Git repository, issue the following commands: ```console +$ ./bootstrap.sh $ ./configure options... $ make $ make install @@ -11,13 +11,6 @@ $ make install Nix requires GNU Make so you may need to invoke `gmake` instead. -When building from the Git repository, these should be preceded by the -command: - -```console -$ ./bootstrap.sh -``` - The installation path can be specified by passing the `--prefix=prefix` to `configure`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index 4a49897e4..bb35c0e9f 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt > **Note** > > You must not add the export and then do the install, as the Nix -> installer will detect the presense of Nix configuration, and abort. +> installer will detect the presence of Nix configuration, and abort. ## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index ae7fd458b..96fa34635 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,18 +1,26 @@ # Installing a Binary Distribution -If you are using Linux or macOS versions up to 10.14 (Mojave), the -easiest way to install Nix is to run the following command: +The easiest way to install Nix is to run the following command: ```console $ sh <(curl -L https://nixos.org/nix/install) ``` -If you're using macOS 10.15 (Catalina) or newer, consult [the macOS -installation instructions](#macos-installation) before installing. +This will run the installer interactively (causing it to explain what +it is doing more explicitly), and perform the default "type" of install +for your platform: +- single-user on Linux +- multi-user on macOS -As of Nix 2.1.0, the Nix installer will always default to creating a -single-user installation, however opting in to the multi-user -installation is highly recommended. + > **Notes on read-only filesystem root in macOS 10.15 Catalina +** + > + > - It took some time to support this cleanly. You may see posts, + > examples, and tutorials using obsolete workarounds. + > - Supporting it cleanly made macOS installs too complex to qualify + > as single-user, so this type is no longer supported on macOS. + +We recommend the multi-user install if it supports your platform and +you can authenticate with `sudo`. # Single User Installation @@ -50,9 +58,9 @@ $ rm -rf /nix The multi-user Nix installation creates system users, and a system service for the Nix daemon. - - Linux running systemd, with SELinux disabled - - - macOS +**Supported Systems** +- Linux running systemd, with SELinux disabled +- macOS You can instruct the installer to perform a multi-user installation on your system: @@ -96,165 +104,28 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist There may also be references to Nix in `/etc/profile`, `/etc/bashrc`, and `/etc/zshrc` which you may remove. -# macOS Installation +# macOS Installation + -Starting with macOS 10.15 (Catalina), the root filesystem is read-only. -This means `/nix` can no longer live on your system volume, and that -you'll need a workaround to install Nix. +We believe we have ironed out how to cleanly support the read-only root +on modern macOS. New installs will do this automatically, and you can +also re-run a new installer to convert your existing setup. -The recommended approach, which creates an unencrypted APFS volume for -your Nix store and a "synthetic" empty directory to mount it over at -`/nix`, is least likely to impair Nix or your system. +This section previously detailed the situation, options, and trade-offs, +but it now only outlines what the installer does. You don't need to know +this to run the installer, but it may help if you run into trouble: -> **Note** -> -> With all separate-volume approaches, it's possible something on your -> system (particularly daemons/services and restored apps) may need -> access to your Nix store before the volume is mounted. Adding -> additional encryption makes this more likely. - -If you're using a recent Mac with a [T2 -chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf), -your drive will still be encrypted at rest (in which case "unencrypted" -is a bit of a misnomer). To use this approach, just install Nix with: - -```console -$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume -``` - -If you don't like the sound of this, you'll want to weigh the other -approaches and tradeoffs detailed in this section. - -> **Note** -> -> All of the known workarounds have drawbacks, but we hope better -> solutions will be available in the future. Some that we have our eye -> on are: -> -> 1. A true firmlink would enable the Nix store to live on the primary -> data volume without the build problems caused by the symlink -> approach. End users cannot currently create true firmlinks. -> -> 2. If the Nix store volume shared FileVault encryption with the -> primary data volume (probably by using the same volume group and -> role), FileVault encryption could be easily supported by the -> installer without requiring manual setup by each user. - -## Change the Nix store path prefix - -Changing the default prefix for the Nix store is a simple approach which -enables you to leave it on your root volume, where it can take full -advantage of FileVault encryption if enabled. Unfortunately, this -approach also opts your device out of some benefits that are enabled by -using the same prefix across systems: - - - Your system won't be able to take advantage of the binary cache - (unless someone is able to stand up and support duplicate caching - infrastructure), which means you'll spend more time waiting for - builds. - - - It's harder to build and deploy packages to Linux systems. - -It would also possible (and often requested) to just apply this change -ecosystem-wide, but it's an intrusive process that has side effects we -want to avoid for now. - -## Use a separate encrypted volume - -If you like, you can also add encryption to the recommended approach -taken by the installer. You can do this by pre-creating an encrypted -volume before you run the installer--or you can run the installer and -encrypt the volume it creates later. - -In either case, adding encryption to a second volume isn't quite as -simple as enabling FileVault for your boot volume. Before you dive in, -there are a few things to weigh: - -1. The additional volume won't be encrypted with your existing - FileVault key, so you'll need another mechanism to decrypt the - volume. - -2. You can store the password in Keychain to automatically decrypt the - volume on boot--but it'll have to wait on Keychain and may not mount - before your GUI apps restore. If any of your launchd agents or apps - depend on Nix-installed software (for example, if you use a - Nix-installed login shell), the restore may fail or break. - - On a case-by-case basis, you may be able to work around this problem - by using `wait4path` to block execution until your executable is - available. - - It's also possible to decrypt and mount the volume earlier with a - login hook--but this mechanism appears to be deprecated and its - future is unclear. - -3. You can hard-code the password in the clear, so that your store - volume can be decrypted before Keychain is available. - -If you are comfortable navigating these tradeoffs, you can encrypt the -volume with something along the lines of: - -```console -$ diskutil apfs enableFileVault /nix -user disk -``` - -## Symlink the Nix store to a custom location - -Another simple approach is using `/etc/synthetic.conf` to symlink the -Nix store to the data volume. This option also enables your store to -share any configured FileVault encryption. Unfortunately, builds that -resolve the symlink may leak the canonical path or even fail. - -Because of these downsides, we can't recommend this approach. - -## Notes on the recommended approach - -This section goes into a little more detail on the recommended approach. -You don't need to understand it to run the installer, but it can serve -as a helpful reference if you run into trouble. - -1. In order to compose user-writable locations into the new read-only - system root, Apple introduced a new concept called `firmlinks`, - which it describes as a "bi-directional wormhole" between two - filesystems. You can see the current firmlinks in - `/usr/share/firmlinks`. Unfortunately, firmlinks aren't (currently?) - user-configurable. - - For special cases like NFS mount points or package manager roots, - [synthetic.conf(5)](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html) - supports limited user-controlled file-creation (of symlinks, and - synthetic empty directories) at `/`. To create a synthetic empty - directory for mounting at `/nix`, add the following line to - `/etc/synthetic.conf` (create it if necessary): - - nix - -2. This configuration is applied at boot time, but you can use - `apfs.util` to trigger creation (not deletion) of new entries - without a reboot: - - ```console - $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B - ``` - -3. Create the new APFS volume with diskutil: - - ```console - $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix - ``` - -4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't - already have other entries, it should look something like: - - # - # Warning - this file should only be modified with vifs(8) - # - # Failure to do so is unsupported and may be destructive. - # - LABEL=Nix\040Store /nix apfs rw,nobrowse - - The nobrowse setting will keep Spotlight from indexing this volume, - and keep it from showing up on your desktop. +- create a new APFS volume for your Nix store +- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" + empty root directory to mount your volume +- specify mount options for the volume in `/etc/fstab` +- if you have FileVault enabled + - generate an encryption password + - put it in your system Keychain + - use it to encrypt the volume +- create a system LaunchDaemon to mount this volume early enough in the + boot process to avoid problems loading or restoring any programs that + need access to your Nix store # Installing a pinned Nix version from a URL diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md new file mode 100644 index 000000000..3d2255b7a --- /dev/null +++ b/doc/manual/src/installation/installing-docker.md @@ -0,0 +1,59 @@ +# Using Nix within Docker + +To run the latest stable release of Nix with Docker run the following command: + +```console +$ docker -ti run nixos/nix +Unable to find image 'nixos/nix:latest' locally +latest: Pulling from nixos/nix +5843afab3874: Pull complete +b52bf13f109c: Pull complete +1e2415612aa3: Pull complete +Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff +Status: Downloaded newer image for nixos/nix:latest +35ca4ada6e96:/# nix --version +nix (Nix) 2.3.12 +35ca4ada6e96:/# exit +``` + +# What is included in Nix' Docker image? + +The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` +(and not with `Dockerfile` as it is usual with Docker images). You can still +base your custom Docker image on it as you would do with any other Docker +image. + +The Docker image is also not based on any other image and includes minimal set +of runtime dependencies that are required to use Nix: + + - pkgs.nix + - pkgs.bashInteractive + - pkgs.coreutils-full + - pkgs.gnutar + - pkgs.gzip + - pkgs.gnugrep + - pkgs.which + - pkgs.curl + - pkgs.less + - pkgs.wget + - pkgs.man + - pkgs.cacert.out + - pkgs.findutils + +# Docker image with the latest development version of Nix + +To get the latest image that was built by [Hydra](https://hydra.nixos.org) run +the following command: + +```console +$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load +$ docker run -ti nix:2.5pre20211105 +``` + +You can also build a Docker image from source yourself: + +```console +$ nix build ./\#hydraJobs.dockerImage.x86_64-linux +$ docker load -i ./result +$ docker run -ti nix:2.5pre20211105 +``` diff --git a/doc/manual/src/installation/installing-source.md b/doc/manual/src/installation/installing-source.md index e52d38a03..09b4e4887 100644 --- a/doc/manual/src/installation/installing-source.md +++ b/doc/manual/src/installation/installing-source.md @@ -1,4 +1,4 @@ # Installing Nix from Source -If no binary package is available, you can download and compile a source -distribution. +If no binary package is available or if you want to hack on Nix, you +can build Nix from its Git repository. diff --git a/doc/manual/src/installation/obtaining-source.md b/doc/manual/src/installation/obtaining-source.md index 0a906e390..da05d243d 100644 --- a/doc/manual/src/installation/obtaining-source.md +++ b/doc/manual/src/installation/obtaining-source.md @@ -1,14 +1,9 @@ -# Obtaining a Source Distribution +# Obtaining the Source -The source tarball of the most recent stable release can be downloaded -from the [Nix homepage](http://nixos.org/nix/download.html). You can -also grab the [most recent development -release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents). - -Alternatively, the most recent sources of Nix can be obtained from its -[Git repository](https://github.com/NixOS/nix). For example, the -following command will check out the latest revision into a directory -called `nix`: +The most recent sources of Nix can be obtained from its [Git +repository](https://github.com/NixOS/nix). For example, the following +command will check out the latest revision into a directory called +`nix`: ```console $ git clone https://github.com/NixOS/nix diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 6825af707..0323a4f55 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -2,9 +2,8 @@ - GNU Autoconf () and the autoconf-archive macro collection - (). These are only - needed to run the bootstrap script, and are not necessary if your - source distribution came with a pre-built `./configure` script. + (). These are + needed to run the bootstrap script. - GNU Make. @@ -26,15 +25,6 @@ available for download from the official repository . - - The bzip2 compressor program and the `libbz2` library. Thus you must - have bzip2 installed, including development headers and libraries. - If your distribution does not provide these, you can obtain bzip2 - from - . - - - `liblzma`, which is provided by XZ Utils. If your distribution does - not provide this, you can get it from . - - cURL and its library. If your distribution does not provide it, you can get it from . @@ -61,8 +51,7 @@ you need version 2.5.35, which is available on [SourceForge](http://lex.sourceforge.net/). Slightly older versions may also work, but ancient versions like the ubiquitous 2.5.4a - won't. Note that these are only required if you modify the parser or - when you are building from the Git repository. + won't. - The `libseccomp` is used to provide syscall filtering on Linux. This is an optional dependency and can be disabled passing a diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index d68445c95..d87487a07 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -76,7 +76,7 @@ there after an upgrade. This means that you can _roll back_ to the old version: ```console -$ nix-env --upgrade some-packages +$ nix-env --upgrade -A nixpkgs.some-package $ nix-env --rollback ``` @@ -122,12 +122,12 @@ Nix expressions generally describe how to build a package from source, so an installation action like ```console -$ nix-env --install firefox +$ nix-env --install -A nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but also all its dependencies (all the way up to the C library and the -compiler) would have to built, at least if they are not already in the +compiler) would have to be built, at least if they are not already in the Nix store. This is a _source deployment model_. For most users, building from source is not very pleasant as it takes far too long. However, Nix can automatically skip building from source and instead diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 9702a29eb..50c6d3c2d 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs, or completely new ones.) You can manually download the latest version of Nixpkgs from -. However, it’s much more +. However, it’s much more convenient to use the Nixpkgs [*channel*](channels.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you @@ -47,41 +47,45 @@ $ nix-channel --update You can view the set of available packages in Nixpkgs: ```console -$ nix-env -qa -aterm-2.2 -bash-3.0 -binutils-2.15 -bison-1.875d -blackdown-1.4.2 -bzip2-1.0.2 +$ nix-env -qaP +nixpkgs.aterm aterm-2.2 +nixpkgs.bash bash-3.0 +nixpkgs.binutils binutils-2.15 +nixpkgs.bison bison-1.875d +nixpkgs.blackdown blackdown-1.4.2 +nixpkgs.bzip2 bzip2-1.0.2 … ``` -The flag `-q` specifies a query operation, and `-a` means that you want +The flag `-q` specifies a query operation, `-a` means that you want to show the “available” (i.e., installable) packages, as opposed to the -installed packages. If you downloaded Nixpkgs yourself, or if you -checked it out from GitHub, then you need to pass the path to your -Nixpkgs tree using the `-f` flag: +installed packages, and `-P` prints the attribute paths that can be used +to unambiguously select a package for installation (listed in the first column). +If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the `-f` flag: ```console -$ nix-env -qaf /path/to/nixpkgs +$ nix-env -qaPf /path/to/nixpkgs +aterm aterm-2.2 +bash bash-3.0 +… ``` where */path/to/nixpkgs* is where you’ve unpacked or checked out Nixpkgs. -You can select specific packages by name: +You can filter the packages by name: ```console -$ nix-env -qa firefox -firefox-34.0.5 -firefox-with-plugins-34.0.5 +$ nix-env -qaP firefox +nixpkgs.firefox-esr firefox-91.3.0esr +nixpkgs.firefox firefox-94.0.1 ``` and using regular expressions: ```console -$ nix-env -qa 'firefox.*' +$ nix-env -qaP 'firefox.*' ``` It is also possible to see the *status* of available packages, i.e., @@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in the system: ```console -$ nix-env -qas +$ nix-env -qaPs … --PS bash-3.0 ---S binutils-2.15 -IPS bison-1.875d +-PS nixpkgs.bash bash-3.0 +--S nixpkgs.binutils binutils-2.15 +IPS nixpkgs.bison bison-1.875d … ``` @@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally. -You can install a package using `nix-env -i`. For instance, +You can install a package using `nix-env -iA`. For instance, ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` -will install the package called `subversion` (which is, of course, the +will install the package called `subversion` from `nixpkgs` channel (which is, of course, the [Subversion version management system](http://subversion.tigris.org/)). > **Note** @@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the > binary cache ; it contains binaries for most > packages in Nixpkgs. Only if no binary is available in the binary > cache, Nix will build the package from source. So if `nix-env -> -i subversion` results in Nix building stuff from source, then either +> -iA nixpkgs.subversion` results in Nix building stuff from source, then either > the package is not built for your platform by the Nixpkgs build > servers, or your version of Nixpkgs is too old or too new. For > instance, if you have a very recent checkout of Nixpkgs, then the @@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the > using a Git checkout of the Nixpkgs tree), you will get binaries for > most packages. -Naturally, packages can also be uninstalled: +Naturally, packages can also be uninstalled. Unlike when installing, you will +need to use the derivation name (though the version part can be omitted), +instead of the attribute path, as `nix-env` does not record which attribute +was used for installing: ```console $ nix-env -e subversion @@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of Nix Packages, you can do: ```console -$ nix-env -u subversion +$ nix-env -uA nixpkgs.subversion ``` This will *only* upgrade Subversion if there is a “newer” version in the diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md index bdc5038fc..ef738794b 100644 --- a/doc/manual/src/package-management/binary-cache-substituter.md +++ b/doc/manual/src/package-management/binary-cache-substituter.md @@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -i nix-serve +$ nix-env -iA nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using `--option extra-binary-caches`, e.g.: ```console -$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ +$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/ ``` The option `extra-binary-caches` tells Nix to use this binary cache in diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/src/package-management/garbage-collection.md index fecb30fd6..29a3b3101 100644 --- a/doc/manual/src/package-management/garbage-collection.md +++ b/doc/manual/src/package-management/garbage-collection.md @@ -44,7 +44,7 @@ collector as follows: $ nix-store --gc ``` -The behaviour of the gargage collector is affected by the +The behaviour of the garbage collector is affected by the `keep-derivations` (default: true) and `keep-outputs` (default: false) options in the Nix configuration file. The defaults will ensure that all derivations that are build-time dependencies of garbage collector roots diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md index fbbfb7320..d1a2580d4 100644 --- a/doc/manual/src/package-management/profiles.md +++ b/doc/manual/src/package-management/profiles.md @@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be what we would obtain if we had done ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ environment is generated based on the current one. For instance, generation 43 was created from generation 42 when we did ```console -$ nix-env -i subversion firefox +$ nix-env -iA nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion +$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md index 6e5e258bc..c59933f61 100644 --- a/doc/manual/src/package-management/ssh-substituter.md +++ b/doc/manual/src/package-management/ssh-substituter.md @@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are available on the server `avalon`: ```console -$ nix-env -i firefox --substituters ssh://alice@avalon +$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 71205923b..b54e73500 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,19 +19,19 @@ to subsequent chapters. channel: ```console - $ nix-env -qa - docbook-xml-4.3 - docbook-xml-4.5 - firefox-33.0.2 - hello-2.9 - libxslt-1.1.28 + $ nix-env -qaP + nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 + nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 + nixpkgs.firefox firefox-33.0.2 + nixpkgs.hello hello-2.9 + nixpkgs.libxslt libxslt-1.1.28 … ``` 1. Install some packages from the channel: ```console - $ nix-env -i hello + $ nix-env -iA nixpkgs.hello ``` This should download pre-built packages; it should not build them diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index f7ab9f6ad..70b715053 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -1,8 +1,539 @@ -# Release 2.4 (202X-XX-XX) +# Release 2.4 (2021-11-01) - - It is now an error to modify the `plugin-files` setting via a - command-line flag that appears after the first non-flag argument - to any command, including a subcommand to `nix`. For example, - `nix-instantiate default.nix --plugin-files ""` must now become - `nix-instantiate --plugin-files "" default.nix`. - - Plugins that add new `nix` subcommands are now actually respected. +This is the first release in more than two years and is the result of +more than 2800 commits from 195 contributors since release 2.3. + +## Highlights + +* Nix's **error messages** have been improved a lot. For instance, + evaluation errors now point out the location of the error: + + ``` + $ nix build + error: undefined variable 'bzip3' + + at /nix/store/449lv242z0zsgwv95a8124xi11sp419f-source/flake.nix:88:13: + + 87| [ curl + 88| bzip3 xz brotli editline + | ^ + 89| openssl sqlite + ``` + +* The **`nix` command** has seen a lot of work and is now almost at + feature parity with the old command-line interface (the `nix-*` + commands). It aims to be [more modern, consistent and pleasant to + use](../contributing/cli-guideline.md) than the old CLI. It is still + marked as experimental but its interface should not change much + anymore in future releases. + +* **Flakes** are a new format to package Nix-based projects in a more + discoverable, composable, consistent and reproducible way. A flake + is just a repository or tarball containing a file named `flake.nix` + that specifies dependencies on other flakes and returns any Nix + assets such as packages, Nixpkgs overlays, NixOS modules or CI + tests. The new `nix` CLI is primarily based around flakes; for + example, a command like `nix run nixpkgs#hello` runs the `hello` + application from the `nixpkgs` flake. + + Flakes are currently marked as experimental. For an introduction, + see [this blog + post](https://www.tweag.io/blog/2020-05-25-flakes/). For detailed + information about flake syntax and semantics, see the [`nix flake` + manual page](../command-ref/new-cli/nix3-flake.md). + +* Nix's store can now be **content-addressed**, meaning that the hash + component of a store path is the hash of the path's + contents. Previously Nix could only build **input-addressed** store + paths, where the hash is computed from the derivation dependency + graph. Content-addressing allows deduplication, early cutoff in + build systems, and unprivileged closure copying. This is still [an + experimental + feature](https://discourse.nixos.org/t/content-addressed-nix-call-for-testers/12881). + +* The Nix manual has been converted into Markdown, making it easier to + contribute. In addition, every `nix` subcommand now has a manual + page, documenting every option. + +* A new setting that allows **experimental features** to be enabled + selectively. This allows us to merge unstable features into Nix more + quickly and do more frequent releases. + +## Other features + +* There are many new `nix` subcommands: + + - `nix develop` is intended to replace `nix-shell`. It has a number + of new features: + + * It automatically sets the output environment variables (such as + `$out`) to writable locations (such as `./outputs/out`). + + * It can store the environment in a profile. This is useful for + offline work. + + * It can run specific phases directly. For instance, `nix develop + --build` runs `buildPhase`. + + - It allows dependencies in the Nix store to be "redirected" to + arbitrary directories using the `--redirect` flag. This is + useful if you want to hack on a package *and* some of its + dependencies at the same time. + + - `nix print-dev-env` prints the environment variables and bash + functions defined by a derivation. This is useful for users of + other shells than bash (especially with `--json`). + + - `nix shell` was previously named `nix run` and is intended to + replace `nix-shell -p`, but without the `stdenv` overhead. It + simply starts a shell where some packages have been added to + `$PATH`. + + - `nix run` (not to be confused with the old subcommand that has + been renamed to `nix shell`) runs an "app", a flake output that + specifies a command to run, or an eponymous program from a + package. For example, `nix run nixpkgs#hello` runs the `hello` + program from the `hello` package in `nixpkgs`. + + - `nix flake` is the container for flake-related operations, such as + creating a new flake, querying the contents of a flake or updating + flake lock files. + + - `nix registry` allows you to query and update the flake registry, + which maps identifiers such as `nixpkgs` to concrete flake URLs. + + - `nix profile` is intended to replace `nix-env`. Its main advantage + is that it keeps track of the provenance of installed packages + (e.g. exactly which flake version a package came from). It also + has some helpful subcommands: + + * `nix profile history` shows what packages were added, upgraded + or removed between each version of a profile. + + * `nix profile diff-closures` shows the changes between the + closures of each version of a profile. This allows you to + discover the addition or removal of dependencies or size + changes. + + **Warning**: after a profile has been updated using `nix profile`, + it is no longer usable with `nix-env`. + + - `nix store diff-closures` shows the differences between the + closures of two store paths in terms of the versions and sizes of + dependencies in the closures. + + - `nix store make-content-addressable` rewrites an arbitrary closure + to make it content-addressed. Such paths can be copied into other + stores without requiring signatures. + + - `nix bundle` uses the [`nix-bundle` + program](https://github.com/matthewbauer/nix-bundle) to convert a + closure into a self-extracting executable. + + - Various other replacements for the old CLI, e.g. `nix store gc`, + `nix store delete`, `nix store repair`, `nix nar dump-path`, `nix + store prefetch-file`, `nix store prefetch-tarball`, `nix key` and + `nix daemon`. + +* Nix now has an **evaluation cache** for flake outputs. For example, + a second invocation of the command `nix run nixpkgs#firefox` will + not need to evaluate the `firefox` attribute because it's already in + the evaluation cache. This is made possible by the hermetic + evaluation model of flakes. + +* The new `--offline` flag disables substituters and causes all + locally cached tarballs and repositories to be considered + up-to-date. + +* The new `--refresh` flag causes all locally cached tarballs and + repositories to be considered out-of-date. + +* Many `nix` subcommands now have a `--json` option to produce + machine-readable output. + +* `nix repl` has a new `:doc` command to show documentation about + builtin functions (e.g. `:doc builtins.map`). + +* Binary cache stores now have an option `index-debug-info` to create + an index of DWARF debuginfo files for use by + [`dwarffs`](https://github.com/edolstra/dwarffs). + +* To support flakes, Nix now has an extensible mechanism for fetching + source trees. Currently it has the following backends: + + * Git repositories + + * Mercurial repositories + + * GitHub and GitLab repositories (an optimisation for faster + fetching than Git) + + * Tarballs + + * Arbitrary directories + + The fetcher infrastructure is exposed via flake input specifications + and via the `fetchTree` built-in. + +* **Languages changes**: the only new language feature is that you can + now have antiquotations in paths, e.g. `./${foo}` instead of `./. + + foo`. + +* **New built-in functions**: + + - `builtins.fetchTree` allows fetching a source tree using any + backends supported by the fetcher infrastructure. It subsumes the + functionality of existing built-ins like `fetchGit`, + `fetchMercurial` and `fetchTarball`. + + - `builtins.getFlake` fetches a flake and returns its output + attributes. This function should not be used inside flakes! Use + flake inputs instead. + + - `builtins.floor` and `builtins.ceil` round a floating-point number + down and up, respectively. + +* Experimental support for recursive Nix. This means that Nix + derivations can now call Nix to build other derivations. This is not + in a stable state yet and not well + [documented](https://github.com/NixOS/nix/commit/c4d7c76b641d82b2696fef73ce0ac160043c18da). + +* The new experimental feature `no-url-literals` disables URL + literals. This helps to implement [RFC + 45](https://github.com/NixOS/rfcs/pull/45). + +* Nix now uses `libarchive` to decompress and unpack tarballs and zip + files, so `tar` is no longer required. + +* The priority of substituters can now be overridden using the + `priority` substituter setting (e.g. `--substituters + 'http://cache.nixos.org?priority=100 daemon?priority=10'`). + +* `nix edit` now supports non-derivation attributes, e.g. `nix edit + .#nixosConfigurations.bla`. + +* The `nix` command now provides command line completion for `bash`, + `zsh` and `fish`. Since the support for getting completions is built + into `nix`, it's easy to add support for other shells. + +* The new `--log-format` flag selects what Nix's output looks like. It + defaults to a terse progress indicator. There is a new + `internal-json` output format for use by other programs. + +* `nix eval` has a new `--apply` flag that applies a function to the + evaluation result. + +* `nix eval` has a new `--write-to` flag that allows it to write a + nested attribute set of string leaves to a corresponding directory + tree. + +* Memory improvements: many operations that add paths to the store or + copy paths between stores now run in constant memory. + +* Many `nix` commands now support the flag `--derivation` to operate + on a `.drv` file itself instead of its outputs. + +* There is a new store called `dummy://` that does not support + building or adding paths. This is useful if you want to use the Nix + evaluator but don't have a Nix store. + +* The `ssh-ng://` store now allows substituting paths on the remote, + as `ssh://` already did. + +* When auto-calling a function with an ellipsis, all arguments are now + passed. + +* New `nix-shell` features: + + - It preserves the `PS1` environment variable if + `NIX_SHELL_PRESERVE_PROMPT` is set. + + - With `-p`, it passes any `--arg`s as Nixpkgs arguments. + + - Support for structured attributes. + +* `nix-prefetch-url` has a new `--executable` flag. + +* On `x86_64` systems, [`x86_64` microarchitecture + levels](https://lwn.net/Articles/844831/) are mapped to additional + system types (e.g. `x86_64-v1-linux`). + +* The new `--eval-store` flag allows you to use a different store for + evaluation than for building or storing the build result. This is + primarily useful when you want to query whether something exists in + a read-only store, such as a binary cache: + + ``` + # nix path-info --json --store https://cache.nixos.org \ + --eval-store auto nixpkgs#hello + ``` + + (Here `auto` indicates the local store.) + +* The Nix daemon has a new low-latency mechanism for copying + closures. This is useful when building on remote stores such as + `ssh-ng://`. + +* Plugins can now register `nix` subcommands. + +## Incompatible changes + +* The `nix` command is now marked as an experimental feature. This + means that you need to add + + ``` + experimental-features = nix-command + ``` + + to your `nix.conf` if you want to use it, or pass + `--extra-experimental-features nix-command` on the command line. + +* The `nix` command no longer has a syntax for referring to packages + in a channel. This means that the following no longer works: + + ```console + nix build nixpkgs.hello # Nix 2.3 + ``` + + Instead, you can either use the `#` syntax to select a package from + a flake, e.g. + + ```console + nix build nixpkgs#hello + ``` + + Or, if you want to use the `nixpkgs` channel in the `NIX_PATH` + environment variable: + + ```console + nix build -f '' hello + ``` + +* The old `nix run` has been renamed to `nix shell`, while there is a + new `nix run` that runs a default command. So instead of + + ```console + nix run nixpkgs.hello -c hello # Nix 2.3 + ``` + + you should use + + ```console + nix shell nixpkgs#hello -c hello + ``` + + or just + + ```console + nix run nixpkgs#hello + ``` + + if the command you want to run has the same name as the package. + +* It is now an error to modify the `plugin-files` setting via a + command-line flag that appears after the first non-flag argument to + any command, including a subcommand to `nix`. For example, + `nix-instantiate default.nix --plugin-files ""` must now become + `nix-instantiate --plugin-files "" default.nix`. + +* We no longer release source tarballs. If you want to build from + source, please build from the tags in the Git repository. + +## Contributors + +This release has contributions from +Adam Höse, +Albert Safin, +Alex Kovar, +Alex Zero, +Alexander Bantyev, +Alexandre Esteves, +Alyssa Ross, +Anatole Lucet, +Anders Kaseorg, +Andreas Rammhold, +Antoine Eiche, +Antoine Martin, +Arnout Engelen, +Arthur Gautier, +aszlig, +Ben Burdette, +Benjamin Hipple, +Bernardo Meurer, +Björn Gohla, +Bjørn Forsman, +Bob van der Linden, +Brian Leung, +Brian McKenna, +Brian Wignall, +Bruce Toll, +Bryan Richter, +Calle Rosenquist, +Calvin Loncaric, +Carlo Nucera, +Carlos D'Agostino, +Chaz Schlarp, +Christian Höppner, +Christian Kampka, +Chua Hou, +Chuck, +Cole Helbling, +Daiderd Jordan, +Dan Callahan, +Dani, +Daniel Fitzpatrick, +Danila Fedorin, +Daniël de Kok, +Danny Bautista, +DavHau, +David McFarland, +Dima, +Domen Kožar, +Dominik Schrempf, +Dominique Martinet, +dramforever, +Dustin DeWeese, +edef, +Eelco Dolstra, +Emilio Karakey, +Emily, +Eric Culp, +Ersin Akinci, +Fabian Möller, +Farid Zakaria, +Federico Pellegrin, +Finn Behrens, +Florian Franzen, +Félix Baylac-Jacqué, +Gabriel Gonzalez, +Geoff Reedy, +Georges Dubus, +Graham Christensen, +Greg Hale, +Greg Price, +Gregor Kleen, +Gregory Hale, +Griffin Smith, +Guillaume Bouchard, +Harald van Dijk, +illustris, +Ivan Zvonimir Horvat, +Jade, +Jake Waksbaum, +jakobrs, +James Ottaway, +Jan Tojnar, +Janne Heß, +Jaroslavas Pocepko, +Jarrett Keifer, +Jeremy Schlatter, +Joachim Breitner, +Joe Hermaszewski, +Joe Pea, +John Ericson, +Jonathan Ringer, +Josef Kemetmüller, +Joseph Lucas, +Jude Taylor, +Julian Stecklina, +Julien Tanguy, +Jörg Thalheim, +Kai Wohlfahrt, +keke, +Keshav Kini, +Kevin Quick, +Kevin Stock, +Kjetil Orbekk, +Krzysztof Gogolewski, +kvtb, +Lars Mühmel, +Leonhard Markert, +Lily Ballard, +Linus Heckemann, +Lorenzo Manacorda, +Lucas Desgouilles, +Lucas Franceschino, +Lucas Hoffmann, +Luke Granger-Brown, +Madeline Haraj, +Marwan Aljubeh, +Mat Marini, +Mateusz Piotrowski, +Matthew Bauer, +Matthew Kenigsberg, +Mauricio Scheffer, +Maximilian Bosch, +Michael Adler, +Michael Bishop, +Michael Fellinger, +Michael Forney, +Michael Reilly, +mlatus, +Mykola Orliuk, +Nathan van Doorn, +Naïm Favier, +ng0, +Nick Van den Broeck, +Nicolas Stig124 Formichella, +Niels Egberts, +Niklas Hambüchen, +Nikola Knezevic, +oxalica, +p01arst0rm, +Pamplemousse, +Patrick Hilhorst, +Paul Opiyo, +Pavol Rusnak, +Peter Kolloch, +Philipp Bartsch, +Philipp Middendorf, +Piotr Szubiakowski, +Profpatsch, +Puck Meerburg, +Ricardo M. Correia, +Rickard Nilsson, +Robert Hensing, +Robin Gloster, +Rodrigo, +Rok Garbas, +Ronnie Ebrin, +Rovanion Luckey, +Ryan Burns, +Ryan Mulligan, +Ryne Everett, +Sam Doshi, +Sam Lidder, +Samir Talwar, +Samuel Dionne-Riel, +Sebastian Ullrich, +Sergei Trofimovich, +Sevan Janiyan, +Shao Cheng, +Shea Levy, +Silvan Mosberger, +Stefan Frijters, +Stefan Jaax, +sternenseemann, +Steven Shaw, +Stéphan Kochen, +SuperSandro2000, +Suraj Barkale, +Taeer Bar-Yam, +Thomas Churchman, +Théophane Hufschmitt, +Timothy DeHerrera, +Timothy Klim, +Tobias Möst, +Tobias Pflug, +Tom Bereknyei, +Travis A. Everett, +Ujjwal Jain, +Vladimír Čunát, +Wil Taylor, +Will Dietz, +Yaroslav Bolyukin, +Yestin L. Harrison, +YI, +Yorick van Pelt, +Yuriy Taraday and +zimbatm. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md new file mode 100644 index 000000000..26c7d2cce --- /dev/null +++ b/doc/manual/src/release-notes/rl-next.md @@ -0,0 +1,7 @@ +# Release 2.5 (2021-XX-XX) + +* Binary cache stores now have a setting `compression-level`. + +* `nix develop` now has a flag `--unpack` to run `unpackPhase`. + +* Lists can now be compared lexicographically using the `<` operator. diff --git a/doc/manual/highlight.pack.js b/doc/manual/theme/highlight.js similarity index 100% rename from doc/manual/highlight.pack.js rename to doc/manual/theme/highlight.js diff --git a/docker.nix b/docker.nix new file mode 100644 index 000000000..2a13c23fb --- /dev/null +++ b/docker.nix @@ -0,0 +1,251 @@ +{ pkgs ? import { } +, lib ? pkgs.lib +, name ? "nix" +, tag ? "latest" +, channelName ? "nixpkgs" +, channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +}: +let + defaultPkgs = with pkgs; [ + nix + bashInteractive + coreutils-full + gnutar + gzip + gnugrep + which + curl + less + wget + man + cacert.out + findutils + ]; + + users = { + + root = { + uid = 0; + shell = "/bin/bash"; + home = "/root"; + gid = 0; + }; + + } // lib.listToAttrs ( + map + ( + n: { + name = "nixbld${toString n}"; + value = { + uid = 30000 + n; + gid = 30000; + groups = [ "nixbld" ]; + description = "Nix build user ${toString n}"; + }; + } + ) + (lib.lists.range 1 32) + ); + + groups = { + root.gid = 0; + nixbld.gid = 30000; + }; + + userToPasswd = ( + k: + { uid + , gid ? 65534 + , home ? "/var/empty" + , description ? "" + , shell ? "/bin/false" + , groups ? [ ] + }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}" + ); + passwdContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToPasswd users)) + ); + + userToShadow = k: { ... }: "${k}:!:1::::::"; + shadowContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToShadow users)) + ); + + # Map groups to members + # { + # group = [ "user1" "user2" ]; + # } + groupMemberMap = ( + let + # Create a flat list of user/group mappings + mappings = ( + builtins.foldl' + ( + acc: user: + let + groups = users.${user}.groups or [ ]; + in + acc ++ map + (group: { + inherit user group; + }) + groups + ) + [ ] + (lib.attrNames users) + ); + in + ( + builtins.foldl' + ( + acc: v: acc // { + ${v.group} = acc.${v.group} or [ ] ++ [ v.user ]; + } + ) + { } + mappings) + ); + + groupToGroup = k: { gid }: + let + members = groupMemberMap.${k} or [ ]; + in + "${k}:x:${toString gid}:${lib.concatStringsSep "," members}"; + groupContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs groupToGroup groups)) + ); + + nixConf = { + sandbox = "false"; + build-users-group = "nixbld"; + trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + }; + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + baseSystem = + let + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixos" { } '' + mkdir $out + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''; + rootEnv = pkgs.buildPackages.buildEnv { + name = "root-profile-env"; + paths = defaultPkgs; + }; + profile = pkgs.buildPackages.runCommand "user-environment" { } '' + mkdir $out + cp -a ${rootEnv}/* $out/ + + cat > $out/manifest.nix < $out/etc/passwd + echo "" >> $out/etc/passwd + + cat $groupContentsPath > $out/etc/group + echo "" >> $out/etc/group + + cat $shadowContentsPath > $out/etc/shadow + echo "" >> $out/etc/shadow + + mkdir -p $out/usr + ln -s /nix/var/nix/profiles/share $out/usr/ + + mkdir -p $out/nix/var/nix/gcroots + + mkdir $out/tmp + + mkdir -p $out/etc/nix + cat $nixConfContentsPath > $out/etc/nix/nix.conf + + mkdir -p $out/root + mkdir -p $out/nix/var/nix/profiles/per-user/root + + ln -s ${profile} $out/nix/var/nix/profiles/default-1-link + ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default $out/root/.nix-profile + + ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link + ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + + mkdir -p $out/root/.nix-defexpr + ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out/root/.nix-channels + + mkdir -p $out/bin $out/usr/bin + ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env + ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh + ''; + +in +pkgs.dockerTools.buildLayeredImageWithNixDb { + + inherit name tag; + + contents = [ baseSystem ]; + + extraCommands = '' + rm -rf nix-support + ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles + ''; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + +} diff --git a/flake.lock b/flake.lock index 06c507e7d..861af1c54 100644 --- a/flake.lock +++ b/flake.lock @@ -3,32 +3,31 @@ "lowdown-src": { "flake": false, "locked": { - "lastModified": 1617481909, - "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=", + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", "owner": "kristapsdz", "repo": "lowdown", - "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", "type": "github" }, "original": { "owner": "kristapsdz", - "ref": "VERSION_0_8_4", "repo": "lowdown", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1614309161, - "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=", + "lastModified": 1632864508, + "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93", + "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-20.09-small", + "ref": "nixos-21.05-small", "type": "indirect" } }, diff --git a/flake.nix b/flake.nix index ebaafb049..ff152ebd6 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,8 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; }; + inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; + inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: @@ -18,7 +18,9 @@ linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" ]; + systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; + + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); @@ -59,6 +61,7 @@ configureFlags = lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" "LDFLAGS=-fuse-ld=gold" ]; @@ -68,7 +71,7 @@ [ buildPackages.bison buildPackages.flex - (lib.getBin buildPackages.lowdown) + (lib.getBin buildPackages.lowdown-nix) buildPackages.mdbook buildPackages.autoconf-archive buildPackages.autoreconfHook @@ -76,10 +79,10 @@ # Tests buildPackages.git - buildPackages.mercurial + buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq ] - ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; buildDeps = [ curl @@ -87,13 +90,12 @@ openssl sqlite libarchive boost - nlohmann_json - lowdown - gmock + lowdown-nix + gtest ] ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isx86_64 libcpuid; + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { @@ -102,7 +104,13 @@ }); propagatedDeps = - [ (boehmgc.override { enableLargeConfig = true; }) + [ ((boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + ]; + })) ]; perlDeps = @@ -119,8 +127,7 @@ '' mkdir -p $out/nix-support - # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix - # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix + # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. tarballPath() { # Remove the store prefix local path=''${1#${builtins.storeDir}/} @@ -133,10 +140,11 @@ substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: - '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + (system: let + tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; + in '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) systems @@ -145,13 +153,15 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { + testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - # Must keep this name short as OSX has a rather strict limit on the - # socket path length, and this name appears in the path of the - # nix-daemon socket used in the tests - name = "nix-tests"; + name = + "nix-tests" + + optionalString + (versionAtLeast daemon.version "2.4pre20211005" && + versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; inherit version; src = self; @@ -170,21 +180,92 @@ installPhase = '' mkdir -p $out ''; - installCheckPhase = "make installcheck"; + installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; }; + binaryTarball = buildPackages: nix: pkgs: let + inherit (pkgs) cacert; + installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; + in + + buildPackages.runCommand "nix-binary-tarball-${version}" + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${pkgs.system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''; + in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. overlay = final: prev: { - # An older version of Nix to test against when using the daemon. - # Currently using `nixUnstable` as the stable one doesn't respect - # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. nixStable = prev.nix; + # Forward from the previous stage as we don’t want it to pick the lowdown override + nixUnstable = prev.nixUnstable; + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -254,9 +335,9 @@ xz pkgs.perl boost - nlohmann_json ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} @@ -270,24 +351,17 @@ }; - lowdown = with final; stdenv.mkDerivation rec { - name = "lowdown-0.8.4"; - - /* - src = fetchurl { - url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz"; - hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw=="; - }; - */ + lowdown-nix = with final; stdenv.mkDerivation rec { + name = "lowdown-0.9.0"; src = lowdown-src; outputs = [ "out" "bin" "dev" ]; - nativeBuildInputs = [ which ]; + nativeBuildInputs = [ buildPackages.which ]; - configurePhase = - '' + configurePhase = '' + ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin @@ -303,92 +377,48 @@ buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); + buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem: + nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); + # Perl bindings for various platforms. perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: + binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); - with nixpkgsFor.${system}; - - let - installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; }; - in - - runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''); + binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: { + name = crossSystem; + value = let + nixpkgsCross = import nixpkgs { + inherit system crossSystem; + overlays = [ self.overlay ]; + }; + in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; + }) crossSystems)); # The first half of the installation script. This is uploaded # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + + # docker image with Nix inside + dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: + let + pkgs = nixpkgsFor.${system}; + image = import ./docker.nix { inherit pkgs; tag = version; }; + in pkgs.runCommand "docker-image-tarball-${version}" + { meta.description = "Docker image with Nix for ${system}"; + } + '' + mkdir -p $out/nix-support + image=$out/image.tar.gz + ln -s ${image} $image + echo "file binary-dist $image" >> $out/nix-support/hydra-build-products + ''); # Line coverage analysis. coverage = @@ -430,6 +460,12 @@ inherit (self) overlay; }; + tests.nssPreload = (import ./tests/nss-preload.nix rec { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }); + tests.githubFlakes = (import ./tests/github-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; @@ -468,25 +504,33 @@ ''; */ + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}; in + pkgs.runCommand "install-tests" { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentUnstable = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixUnstable + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; - installTests = - let pkgs = nixpkgsFor.${system}; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"; - }); + installTests = self.hydraJobs.installTests.${system}; + } // (if system == "x86_64-linux" then { + dockerImage = self.hydraJobs.dockerImage.${system}; + } else {})); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; - } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { + } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { @@ -524,8 +568,49 @@ stripAllList = ["bin"]; strictDeps = true; + + hardeningDisable = [ "pie" ]; }; - }); + } // builtins.listToAttrs (map (crossSystem: { + name = "nix-${crossSystem}"; + value = let + nixpkgsCross = import nixpkgs { + inherit system crossSystem; + overlays = [ self.overlay ]; + }; + in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation { + name = "nix-${version}"; + + src = self; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ]; + + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ propagatedDeps; + + configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ]; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + }; + }) crossSystems))); defaultPackage = forAllSystems (system: self.packages.${system}.nix); diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 6f3882a12..18ab33424 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -19,6 +19,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; +my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1"; + # FIXME: cut&paste from nixos-channel-scripts. sub fetch { my ($url, $type) = @_; @@ -35,16 +37,18 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); +my $flakeUrl = $evalInfo->{flake} or die; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); +my $nixRev = $flakeInfo->{revision} or die; -my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die; +my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +#print Dumper($buildInfo); -my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json')); - -my $releaseName = $tarballInfo->{releasename}; +my $releaseName = $buildInfo->{nixname}; $releaseName =~ /nix-(.*)$/ or die; my $version = $1; -print STDERR "Nix revision is $nixRev, version is $version\n"; +print STDERR "Flake URL is $flakeUrl, Nix revision is $nixRev, version is $version\n"; my $releaseDir = "nix/$releaseName"; @@ -83,12 +87,12 @@ sub downloadFile { if (!-e $tmpFile) { print STDERR "downloading $srcFile to $tmpFile...\n"; - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0 + system("NIX_REMOTE=https://cache.nixos.org/ nix store cat '$srcFile' > '$tmpFile'") == 0 or die "unable to fetch $srcFile\n"; } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`; + my $sha256_actual = `nix hash file --base16 --type sha256 '$tmpFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; @@ -104,12 +108,13 @@ sub downloadFile { return $sha256_expected; } -downloadFile("tarball", "2"); # .tar.bz2 -my $tarballHash = downloadFile("tarball", "3"); # .tar.xz downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarball.aarch64-darwin", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); downloadFile("installerScript", "1"); for my $fn (glob "$tmpDir/*") { @@ -131,53 +136,38 @@ for my $fn (glob "$tmpDir/*") { } } -exit if $version =~ /pre/; +# Update nix-fallback-paths.nix. +if ($isLatest) { + system("cd $nixpkgsDir && git pull") == 0 or die; -# Update Nixpkgs in a very hacky way. -system("cd $nixpkgsDir && git pull") == 0 or die; -my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName; -my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash; -print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n"; - -my $fn = "$nixpkgsDir/pkgs/tools/package-management/nix/default.nix"; -my $oldFile = read_file($fn); -$oldFile =~ s/$oldName/"$releaseName"/g; -$oldFile =~ s/$oldHash/"$tarballHash"/g; -write_file($fn, $oldFile); - -$oldName =~ s/nix-//g; -$oldName =~ s/"//g; - -sub getStorePath { - my ($jobName) = @_; - my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - for my $product (values %{$buildInfo->{buildproducts}}) { - next unless $product->{type} eq "nix-build"; - next if $product->{path} =~ /[a-z]+$/; - return $product->{path}; + sub getStorePath { + my ($jobName) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; } - die; + + write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", + "{\n" . + " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + "}\n"); + + system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; } -write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", - "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - "}\n"); - -system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; - # Update the "latest" symlink. $channelsBucket->add_key( "nix-latest/install", "", { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" }) - or die $channelsBucket->err . ": " . $channelsBucket->errstr; + or die $channelsBucket->err . ": " . $channelsBucket->errstr + if $isLatest; # Tag the release in Git. chdir("/home/eelco/Dev/nix-pristine") or die; system("git remote update origin") == 0 or die; system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die; system("git push --tags") == 0 or die; -system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die; +system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest; diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish new file mode 100644 index 000000000..bedbefaf8 --- /dev/null +++ b/misc/fish/completion.fish @@ -0,0 +1,37 @@ +function _nix_complete + # Get the current command up to a cursor. + # - Behaves correctly even with pipes and nested in commands like env. + # - TODO: Returns the command verbatim (does not interpolate variables). + # That might not be optimal for arguments like -f. + set -l nix_args (commandline --current-process --tokenize --cut-at-cursor) + # --cut-at-cursor with --tokenize removes the current token so we need to add it separately. + # https://github.com/fish-shell/fish-shell/issues/7375 + # Can be an empty string. + set -l current_token (commandline --current-token --cut-at-cursor) + + # Nix wants the index of the argv item to complete but the $nix_args variable + # also contains the program name (argv[0]) so we would need to subtract 1. + # But the variable also misses the current token so it cancels out. + set -l nix_arg_to_complete (count $nix_args) + + env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token +end + +function _nix_accepts_files + set -l response (_nix_complete) + # First line is either filenames or no-filenames. + test $response[1] = 'filenames' +end + +function _nix + set -l response (_nix_complete) + # Skip the first line since it handled by _nix_accepts_files. + # Tail lines each contain a command followed by a tab character and, optionally, a description. + # This is also the format fish expects. + string collect -- $response[2..-1] +end + +# Disable file path completion if paths do not belong in the current context. +complete --command nix --condition 'not _nix_accepts_files' --no-files + +complete --command nix --arguments '(_nix)' diff --git a/misc/fish/local.mk b/misc/fish/local.mk new file mode 100644 index 000000000..ece899fc3 --- /dev/null +++ b/misc/fish/local.mk @@ -0,0 +1 @@ +$(eval $(call install-file-as, $(d)/completion.fish, $(datarootdir)/fish/vendor_completions.d/nix.fish, 0644)) diff --git a/misc/launchd/local.mk b/misc/launchd/local.mk index 0ba722efb..a39188fe6 100644 --- a/misc/launchd/local.mk +++ b/misc/launchd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons)) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index c334639e2..f1b439840 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -19,7 +19,7 @@ /bin/sh -c - /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && exec /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 785db52a4..1fa037485 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index 5071676dc..2fbfb29b9 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index d4df6447e..a902e37dc 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -1,3 +1,5 @@ +#compdef nix + function _nix() { local ifs_bk="$IFS" local input=("${(Q)words[@]}") @@ -18,4 +20,4 @@ function _nix() { _describe 'nix' suggestions } -compdef _nix nix +_nix "$@" diff --git a/misc/zsh/local.mk b/misc/zsh/local.mk new file mode 100644 index 000000000..418fb1377 --- /dev/null +++ b/misc/zsh/local.mk @@ -0,0 +1 @@ +$(eval $(call install-file-as, $(d)/completion.zsh, $(datarootdir)/zsh/site-functions/_nix, 0644)) diff --git a/mk/lib.mk b/mk/lib.mk index 975102531..92f0983d5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,8 +10,25 @@ bin-scripts := noinst-scripts := man-pages := install-tests := -OS = $(shell uname -s) +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + endif +endif # Hack to define a literal space. space := @@ -50,16 +67,16 @@ endif BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE else GLOBAL_CFLAGS += -fPIC GLOBAL_CXXFLAGS += -fPIC endif - ifneq ($(OS), Darwin) - ifneq ($(OS), SunOS) - ifneq ($(OS), FreeBSD) + ifndef HOST_DARWIN + ifndef HOST_SOLARIS + ifndef HOST_FREEBSD GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries endif endif diff --git a/mk/libraries.mk b/mk/libraries.mk index 7c0e4f100..ffd7b5610 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -1,9 +1,9 @@ libs-list := -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN SO_EXT = dylib else - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN SO_EXT = dll else SO_EXT = so @@ -59,7 +59,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -73,18 +73,18 @@ define build-library ifeq ($(BUILD_SHARED_LIBS), 1) ifdef $(1)_ALLOW_UNDEFINED - ifeq ($(OS), Darwin) + ifdef HOST_DARWIN $(1)_LDFLAGS += -undefined suppress -flat_namespace endif else - ifneq ($(OS), Darwin) - ifneq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifndef HOST_DARWIN + ifndef HOST_CYGWIN $(1)_LDFLAGS += -Wl,-z,defs endif endif endif - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT) endif @@ -93,7 +93,7 @@ define build-library $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) endif $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) @@ -108,7 +108,7 @@ define build-library $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN ifeq ($(SET_RPATH_TO_LIBS), 1) $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) else @@ -125,8 +125,8 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? - $(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o + $$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + $$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) diff --git a/mk/tests.mk b/mk/tests.mk index 21bdc5748..a2e30a378 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -13,3 +13,7 @@ define run-install-test endef .PHONY: check installcheck + +print-top-help += \ + echo " check: Run unit tests"; \ + echo " installcheck: Run functional tests"; diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 50db4783c..538244594 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -8,10 +8,15 @@ endif libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) -libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ifeq ($(OS), Darwin) +ifdef HOST_LINUX +libnixrust_LDFLAGS_USE += -ldl +libnixrust_LDFLAGS_USE_INSTALLED += -ldl +endif + +ifdef HOST_DARWIN libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" else libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) @@ -26,7 +31,7 @@ $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml $(libnixrust_INSTALL_PATH): $(libnixrust_PATH) $(target-gen) cp $^ $@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN install_name_tool -id $@ $@ endif @@ -35,7 +40,7 @@ clean: clean-rust clean-rust: $(suppress) rm -rfv nix-rust/target -ifneq ($(OS), Darwin) +ifndef HOST_DARWIN check: rust-tests rust-tests: diff --git a/perl/Makefile b/perl/Makefile index 259ed7dc3..708f86882 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++17 +GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src -include Makefile.config diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index eccfbd9f6..d856de3ad 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ diff --git a/perl/configure.ac b/perl/configure.ac index 85183c005..eb65ac17b 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -7,6 +7,8 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX +AC_CANONICAL_HOST + # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index f7c6f2484..508a15e15 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -1,6 +1,7 @@ package Nix::Config; use MIME::Base64; +use Nix::Store; $version = "@PACKAGE_VERSION@"; diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index 179f1dc90..3e4bbee0a 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -22,6 +22,7 @@ our @EXPORT = qw( derivationFromPath addTempRoot getBinDir getStoreDir + queryRawRealisation ); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index ad9042a2a..edbf12f7c 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -15,6 +15,7 @@ #include "crypto.hh" #include +#include using namespace nix; @@ -120,6 +121,18 @@ SV * queryPathInfo(char * path, int base32) croak("%s", e.what()); } +SV * queryRawRealisation(char * outputId) + PPCODE: + try { + auto realisation = store()->queryRealisation(DrvOutput::parse(outputId)); + if (realisation) + XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); + else + XPUSHs(sv_2mortal(newSVpv("", 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + SV * queryPathFromHashPart(char * hashPart) PPCODE: diff --git a/perl/local.mk b/perl/local.mk index b13d4c0d6..0eae651d8 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -28,7 +28,7 @@ Store_CXXFLAGS = \ Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) -ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) +ifdef HOST_CYGWIN archlib = $(shell perl -E 'use Config; print $$Config{archlib};') libperl = $(shell perl -E 'use Config; print $$Config{libperl};') Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 32fa577a8..334b75045 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -1,33 +1,262 @@ -#!/bin/sh -set -e +#!/usr/bin/env bash +set -eu +set -o pipefail -root_disk() { - diskutil info -plist / -} +# I'm a little agnostic on the choices, but supporting a wide +# slate of uses for now, including: +# - import-only: `. create-darwin-volume.sh no-main[ ...]` +# - legacy: `./create-darwin-volume.sh` or `. create-darwin-volume.sh` +# (both will run main()) +# - external alt-routine: `./create-darwin-volume.sh no-main func[ ...]` +if [ "${1-}" = "no-main" ]; then + shift + readonly _CREATE_VOLUME_NO_MAIN=1 +else + readonly _CREATE_VOLUME_NO_MAIN=0 + # declare some things we expect to inherit from install-multi-user + # I don't love this (because it's a bit of a kludge). + # + # CAUTION: (Dec 19 2020) + # This is a stopgap. It doesn't cover the full slate of + # identifiers we inherit--just those necessary to: + # - avoid breaking direct invocations of this script (here/now) + # - avoid hard-to-reverse structural changes before the call to rm + # single-user support is verified + # + # In the near-mid term, I (personally) think we should: + # - decide to deprecate the direct call and add a notice + # - fold all of this into install-darwin-multi-user.sh + # - intentionally remove the old direct-invocation form (kill the + # routine, replace this script w/ deprecation notice and a note + # on the remove-after date) + # + readonly NIX_ROOT="${NIX_ROOT:-/nix}" -# i.e., "disk1" + _sudo() { + shift # throw away the 'explanation' + /usr/bin/sudo "$@" + } + failure() { + if [ "$*" = "" ]; then + cat + else + echo "$@" + fi + exit 1 + } + task() { + echo "$@" + } +fi + +# usually "disk1" root_disk_identifier() { - diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # For performance (~10ms vs 280ms) I'm parsing 'diskX' from stat output + # (~diskXsY)--but I'm retaining the more-semantic approach since + # it documents intent better. + # /usr/sbin/diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # + local special_device + special_device="$(/usr/bin/stat -f "%Sd" /)" + echo "${special_device%s[0-9]*}" } -find_nix_volume() { - diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true +# make it easy to play w/ 'Case-sensitive APFS' +readonly NIX_VOLUME_FS="${NIX_VOLUME_FS:-APFS}" +readonly NIX_VOLUME_LABEL="${NIX_VOLUME_LABEL:-Nix Store}" +# Strongly assuming we'll make a volume on the device / is on +# But you can override NIX_VOLUME_USE_DISK to create it on some other device +readonly NIX_VOLUME_USE_DISK="${NIX_VOLUME_USE_DISK:-$(root_disk_identifier)}" +NIX_VOLUME_USE_SPECIAL="${NIX_VOLUME_USE_SPECIAL:-}" +NIX_VOLUME_USE_UUID="${NIX_VOLUME_USE_UUID:-}" +readonly NIX_VOLUME_MOUNTD_DEST="${NIX_VOLUME_MOUNTD_DEST:-/Library/LaunchDaemons/org.nixos.darwin-store.plist}" + +if /usr/bin/fdesetup isactive >/dev/null; then + test_filevault_in_use() { return 0; } + # no readonly; we may modify if user refuses from cure_volume + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-1}" +else + test_filevault_in_use() { return 1; } + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-0}" +fi + +should_encrypt_volume() { + test_filevault_in_use && (( NIX_VOLUME_DO_ENCRYPT == 1 )) +} + +substep() { + printf " %s\n" "" "- $1" "" "${@:2}" +} + + +volumes_labeled() { + local label="$1" + xsltproc --novalid --stringparam label "$label" - <(/usr/sbin/ioreg -ra -c "AppleAPFSVolume") <<'EOF' + + + + + + + + = + + + + +EOF + # I cut label out of the extracted values, but here it is for reference: + # + # = +} + +right_disk() { + local volume_special="$1" # (i.e., disk1s7) + [[ "$volume_special" == "$NIX_VOLUME_USE_DISK"s* ]] +} + +right_volume() { + local volume_special="$1" # (i.e., disk1s7) + # if set, it must match; otherwise ensure it's on the right disk + if [ -z "$NIX_VOLUME_USE_SPECIAL" ]; then + if right_disk "$volume_special"; then + NIX_VOLUME_USE_SPECIAL="$volume_special" # latch on + return 0 + else + return 1 + fi + else + [ "$volume_special" = "$NIX_VOLUME_USE_SPECIAL" ] + fi +} + +right_uuid() { + local volume_uuid="$1" + # if set, it must match; otherwise allow + if [ -z "$NIX_VOLUME_USE_UUID" ]; then + NIX_VOLUME_USE_UUID="$volume_uuid" # latch on + return 0 + else + [ "$volume_uuid" = "$NIX_VOLUME_USE_UUID" ] + fi +} + +cure_volumes() { + local found volume special uuid + # loop just in case they have more than one volume + # (nothing stops you from doing this) + for volume in $(volumes_labeled "$NIX_VOLUME_LABEL"); do + # CAUTION: this could (maybe) be a more normal read + # loop like: + # while IFS== read -r special uuid; do + # # ... + # done <<<"$(volumes_labeled "$NIX_VOLUME_LABEL")" + # + # I did it with for to skirt a problem with the obvious + # pattern replacing stdin and causing user prompts + # inside (which also use read and access stdin) to skip + # + # If there's an existing encrypted volume we can't find + # in keychain, the user never gets prompted to delete + # the volume, and the install fails. + # + # If you change this, a human needs to test a very + # specific scenario: you already have an encrypted + # Nix Store volume, and have deleted its credential + # from keychain. Ensure the script asks you if it can + # delete the volume, and then prompts for your sudo + # password to confirm. + # + # shellcheck disable=SC1097 + IFS== read -r special uuid <<< "$volume" + # take the first one that's on the right disk + if [ -z "${found:-}" ]; then + if right_volume "$special" && right_uuid "$uuid"; then + cure_volume "$special" "$uuid" + found="${special} (${uuid})" + else + warning < + # Cryptographic user for (1 found) + # Cryptographic users for (2 found) + /usr/sbin/diskutil apfs listCryptoUsers -plist "$volume_special" | /usr/bin/grep -q APFSCryptoUserUUID } test_fstab() { - grep -q "/nix apfs rw" /etc/fstab 2>/dev/null + /usr/bin/grep -q "$NIX_ROOT apfs rw" /etc/fstab 2>/dev/null } -test_nix_symlink() { - [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null +test_nix_root_is_symlink() { + [ -L "$NIX_ROOT" ] } -test_synthetic_conf() { - grep -q "^nix$" /etc/synthetic.conf 2>/dev/null +test_synthetic_conf_either(){ + /usr/bin/grep -qE "^${NIX_ROOT:1}($|\t.{3,}$)" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_mountable() { + /usr/bin/grep -q "^${NIX_ROOT:1}$" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_symlinked() { + /usr/bin/grep -qE "^${NIX_ROOT:1}\t.{3,}$" /etc/synthetic.conf 2>/dev/null +} + +test_nix_volume_mountd_installed() { + test -e "$NIX_VOLUME_MOUNTD_DEST" +} + +# current volume password +test_keychain_by_uuid() { + local volume_uuid="$1" + # Note: doesn't need sudo just to check; doesn't output pw + security find-generic-password -s "$volume_uuid" &>/dev/null +} + +get_volume_pass() { + local volume_uuid="$1" + _sudo \ + "to confirm keychain has a password that unlocks this volume" \ + security find-generic-password -s "$volume_uuid" -w +} + +verify_volume_pass() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid" +} + +volume_pass_works() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + get_volume_pass "$volume_uuid" | verify_volume_pass "$volume_special" "$volume_uuid" } # Create the paths defined in synthetic.conf, saving us a reboot. -create_synthetic_objects(){ +create_synthetic_objects() { # Big Sur takes away the -B flag we were using and replaces it # with a -t flag that appears to do the same thing (but they # don't behave exactly the same way in terms of return values). @@ -41,129 +270,578 @@ create_synthetic_objects(){ } test_nix() { - test -d "/nix" + test -d "$NIX_ROOT" } -test_t2_chip_present(){ - # Use xartutil to see if system has a t2 chip. - # - # This isn't well-documented on its own; until it is, - # let's keep track of knowledge/assumptions. - # - # Warnings: - # - Don't search "xart" if porn will cause you trouble :) - # - Other xartutil flags do dangerous things. Don't run them - # naively. If you must, search "xartutil" first. - # - # Assumptions: - # - the "xART session seeds recovery utility" - # appears to interact with xartstorageremoted - # - `sudo xartutil --list` lists xART sessions - # and their seeds and exits 0 if successful. If - # not, it exits 1 and prints an error such as: - # xartutil: ERROR: No supported link to the SEP present - # - xART sessions/seeds are present when a T2 chip is - # (and not, otherwise) - # - the presence of a T2 chip means a newly-created - # volume on the primary drive will be - # encrypted at rest - # - all together: `sudo xartutil --list` - # should exit 0 if a new Nix Store volume will - # be encrypted at rest, and exit 1 if not. - sudo xartutil --list >/dev/null 2>/dev/null +test_voldaemon() { + test -f "$NIX_VOLUME_MOUNTD_DEST" } -test_filevault_in_use() { - fdesetup isactive >/dev/null +generate_mount_command() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid mountpoint cmd=() + printf -v volume_uuid "%q" "$2" + printf -v mountpoint "%q" "$NIX_ROOT" + + case "$cmd_type" in + encrypted) + cmd=(/bin/sh -c "/usr/bin/security find-generic-password -s '$volume_uuid' -w | /usr/sbin/diskutil apfs unlockVolume '$volume_uuid' -mountpoint '$mountpoint' -stdinpassphrase");; + unencrypted) + cmd=(/usr/sbin/diskutil mount -mountPoint "$mountpoint" "$volume_uuid");; + *) + failure "Invalid first arg $cmd_type to generate_mount_command";; + esac + + printf " %s\n" "${cmd[@]}" } -# use after error msg for conditions we don't understand -suggest_report_error(){ - # ex "error: something sad happened :(" >&2 - echo " please report this @ https://github.com/nixos/nix/issues" >&2 +generate_mount_daemon() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid="$2" + cat < + + + + RunAtLoad + + Label + org.nixos.darwin-store + ProgramArguments + +$(generate_mount_command "$cmd_type" "$volume_uuid") + + + +EOF } -main() { - ( - echo "" - echo " ------------------------------------------------------------------ " - echo " | This installer will create a volume for the nix store and |" - echo " | configure it to mount at /nix. Follow these steps to uninstall. |" - echo " ------------------------------------------------------------------ " - echo "" - echo " 1. Remove the entry from fstab using 'sudo vifs'" - echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" - echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" - echo "" - ) >&2 +_eat_bootout_err() { + /usr/bin/grep -v "Boot-out failed: 36: Operation now in progress" +} - if test_nix_symlink; then - echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 - echo " /nix -> $(readlink "/nix")" >&2 - exit 2 +# TODO: remove with --uninstall? +uninstall_launch_daemon_directions() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + substep "Uninstall LaunchDaemon $daemon_label" \ + " sudo launchctl bootout system/$daemon_label" \ + " sudo rm $daemon_plist" +} + +uninstall_launch_daemon_prompt() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + local reason_for_daemon="$3" + cat < >(_eat_bootout_err >&2) || true + # this can "fail" with a message like: + # Boot-out failed: 36: Operation now in progress + _sudo "to remove the daemon definition" rm "$daemon_plist" + fi +} + +nix_volume_mountd_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" +} + +nix_volume_mountd_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" \ + "mount your Nix volume" +} + +# TODO: move nix_daemon to install-darwin-multi-user if/when uninstall_launch_daemon_prompt moves up to install-multi-user +nix_daemon_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" \ + "run the nix-daemon" +} + +# TODO: remove with --uninstall? +nix_daemon_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" +} + + +# TODO: remove with --uninstall? +synthetic_conf_uninstall_directions() { + # :1 to strip leading slash + substep "Remove ${NIX_ROOT:1} from /etc/synthetic.conf" \ + " If nix is the only entry: sudo rm /etc/synthetic.conf" \ + " Otherwise: sudo /usr/bin/sed -i '' -e '/^${NIX_ROOT:1}$/d' /etc/synthetic.conf" +} + +synthetic_conf_uninstall_prompt() { + cat < "$SCRATCH/synthetic.conf.edit" + + if test_synthetic_conf_symlinked; then + warning <&2 - echo nix | sudo tee -a /etc/synthetic.conf - if ! test_synthetic_conf; then - echo "error: failed to configure synthetic.conf;" >&2 - suggest_report_error - exit 1 + # ask to rm if this left the file empty aside from comments, else edit + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/synthetic.conf.edit") &>/dev/null; then + if confirm_rm "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 + fi + else + if confirm_edit "$SCRATCH/synthetic.conf.edit" "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 fi fi + # fallback instructions + echo "Manually remove nix from /etc/synthetic.conf" + return 1 +} - if ! test_nix; then - echo "Creating mountpoint for /nix..." >&2 - create_synthetic_objects # the ones we defined in synthetic.conf - if ! test_nix; then - sudo mkdir -p /nix 2>/dev/null || true +add_nix_vol_fstab_line() { + local uuid="$1" + # shellcheck disable=SC1003,SC2026 + local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" + shift + EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <multi-user reinstalls, which may cover this) + # + # I'm not sure if it's safe to approach this way? + # + # I think I think the most-proper way to test for it is: + # diskutil info -plist "$NIX_VOLUME_LABEL" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1][name()='true']" -; echo $? + # + # There's also `sudo /usr/sbin/vsdbutil -c /path` (which is much faster, but is also + # deprecated and needs minor parsing). + # + # If no one finds a problem with doing so, I think the simplest approach + # is to just eagerly set this. I found a few imperative approaches: + # (diskutil enableOwnership, ~100ms), a cheap one (/usr/sbin/vsdbutil -a, ~40-50ms), + # a very cheap one (append the internal format to /var/db/volinfo.database). + # + # But vsdbutil's deprecation notice suggests using fstab, so I want to + # give that a whirl first. + # + # TODO: when this is workable, poke infinisil about reproducing the issue + # and confirming this fix? +} + +delete_nix_vol_fstab_line() { + # TODO: I'm scaffolding this to handle the new nix volumes + # but it might be nice to generalize a smidge further to + # go ahead and set up a pattern for curing "old" things + # we no longer do? + EDITOR="/usr/bin/patch" _sudo "to cut nix from fstab" "$@" < <(/usr/bin/diff /etc/fstab <(/usr/bin/grep -v "$NIX_ROOT apfs rw" /etc/fstab)) + # leaving some parts out of the grep; people may fiddle this a little? +} + +# TODO: hope to remove with --uninstall +fstab_uninstall_directions() { + substep "Remove ${NIX_ROOT} from /etc/fstab" \ + " If nix is the only entry: sudo rm /etc/fstab" \ + " Otherwise, run 'sudo /usr/sbin/vifs' to remove the nix line" +} + +fstab_uninstall_prompt() { + cat </dev/null + + # if the patch test edit, minus comment lines, is equal to empty (:) + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/fstab.edit") &>/dev/null; then + # this edit would leave it empty; propose deleting it + if confirm_rm "/etc/fstab"; then + return 0 + else + echo "Remove nix from /etc/fstab (or remove the file)" fi - if ! test_nix; then - echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 - suggest_report_error - exit 1 + else + echo "I might be able to help you make this edit. Here's the diff:" + if ! _diff "/etc/fstab" "$SCRATCH/fstab.edit" && ui_confirm "Does the change above look right?"; then + delete_nix_vol_fstab_line /usr/sbin/vifs + else + echo "Remove nix from /etc/fstab (or remove the file)" fi fi +} - disk="$(root_disk_identifier)" - volume=$(find_nix_volume "$disk") - if [ -z "$volume" ]; then - echo "Creating a Nix Store volume..." >&2 +remove_volume() { + local volume_special="$1" # (i.e., disk1s7) + _sudo "to unmount the Nix volume" \ + /usr/sbin/diskutil unmount force "$volume_special" || true # might not be mounted + _sudo "to delete the Nix volume" \ + /usr/sbin/diskutil apfs deleteVolume "$volume_special" +} - if test_filevault_in_use; then - # TODO: Not sure if it's in-scope now, but `diskutil apfs list` - # shows both filevault and encrypted at rest status, and it - # may be the more semantic way to test for this? It'll show - # `FileVault: No (Encrypted at rest)` - # `FileVault: No` - # `FileVault: Yes (Unlocked)` - # and so on. - if test_t2_chip_present; then - echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 - echo " is only encrypted at rest." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 +# aspiration: robust enough to both fix problems +# *and* update older darwin volumes +cure_volume() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + header "Found existing Nix volume" + row " special" "$volume_special" + row " uuid" "$volume_uuid" + + if volume_encrypted "$volume_special"; then + row "encrypted" "yes" + if volume_pass_works "$volume_special" "$volume_uuid"; then + NIX_VOLUME_DO_ENCRYPT=0 + ok "Found a working decryption password in keychain :)" + echo "" + else + # - this is a volume we made, and + # - the user encrypted it on their own + # - something deleted the credential + # - this is an old or BYO volume and the pw + # just isn't somewhere we can find it. + # + # We're going to explain why we're freaking out + # and prompt them to either delete the volume + # (requiring a sudo auth), or abort to fix + warning <&2 - echo " FileVault encrypted, but encryption-at-rest is not available." >&2 - echo " Manually create a volume for the store and re-run this script." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 - exit 1 + # TODO: this is a good design case for a warn-and + # remind idiom... + failure <&2 - fi - - if ! test_fstab; then - echo "Configuring /etc/fstab..." >&2 - label=$(echo "$volume" | sed 's/ /\\040/g') - # shellcheck disable=SC2209 - printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + row "encrypted" "no" fi } -main "$@" +remove_volume_artifacts() { + if test_synthetic_conf_either; then + # NIX_ROOT is in synthetic.conf + if synthetic_conf_uninstall_prompt; then + # TODO: moot until we tackle uninstall, but when we're + # actually uninstalling, we should issue: + # reminder "macOS will clean up the empty mount-point directory at $NIX_ROOT on reboot." + : + fi + fi + if test_fstab; then + fstab_uninstall_prompt + fi + + if test_nix_volume_mountd_installed; then + nix_volume_mountd_uninstall_prompt + fi +} + +setup_synthetic_conf() { + if test_nix_root_is_symlink; then + if ! test_synthetic_conf_symlinked; then + failure >&2 < $(readlink "$NIX_ROOT")). +Please remove it. If nix is in /etc/synthetic.conf, remove it and reboot. +EOF + fi + fi + if ! test_synthetic_conf_mountable; then + task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2 + # technically /etc/synthetic.d/nix is supported in Big Sur+ + # but handling both takes even more code... + _sudo "to add Nix to /etc/synthetic.conf" \ + /usr/bin/ex /etc/synthetic.conf <&2 + fi + create_synthetic_objects + if ! test_nix; then + failure >&2 <&2 + add_nix_vol_fstab_line "$volume_uuid" /usr/sbin/vifs + fi +} + +encrypt_volume() { + local volume_uuid="$1" + local volume_label="$2" + local password + # Note: mount/unmount are late additions to support the right order + # of operations for creating the volume and then baking its uuid into + # other artifacts; not as well-trod wrt to potential errors, race + # conditions, etc. + + /usr/sbin/diskutil mount "$volume_label" + + password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)" + _sudo "to add your Nix volume's password to Keychain" \ + /usr/bin/security -i </dev/null; do + : + done +} + +setup_volume() { + local use_special use_uuid profile_packages + task "Creating a Nix volume" >&2 + + use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}" + + _sudo "to ensure the Nix volume is not mounted" \ + /usr/sbin/diskutil unmount force "$use_special" || true # might not be mounted + + use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")} + + setup_fstab "$use_uuid" + + if should_encrypt_volume; then + encrypt_volume "$use_uuid" "$NIX_VOLUME_LABEL" + setup_volume_daemon "encrypted" "$use_uuid" + # TODO: might be able to save ~60ms by caching or setting + # this somewhere rather than re-checking here. + elif volume_encrypted "$use_special"; then + setup_volume_daemon "encrypted" "$use_uuid" + else + setup_volume_daemon "unencrypted" "$use_uuid" + fi + + await_volume + + if [ "$(/usr/sbin/diskutil info -plist "$NIX_ROOT" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "" ]; then + _sudo "to set enableOwnership (enabling users to own files)" \ + /usr/sbin/diskutil enableOwnership "$NIX_ROOT" + fi + + # TODO: below is a vague kludge for now; I just don't know + # what if any safe action there is to take here. Also, the + # reminder isn't very helpful. + # I'm less sure where this belongs, but it also wants mounted, pre-install + if type -p nix-env; then + profile_packages="$(nix-env --query --installed)" + # TODO: can probably do below faster w/ read + # intentionally unquoted string to eat whitespace in wc output + # shellcheck disable=SC2046,SC2059 + if ! [ $(printf "$profile_packages" | /usr/bin/wc -l) = "0" ]; then + reminder <&2 + _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <&2 + + setup_darwin_volume + } + + main "$@" +fi diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index f6575ae2f..96eba8310 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,59 +3,110 @@ set -eu set -o pipefail -readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +# create by default; set 0 to DIY, use a symlink, etc. +readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default NIX_FIRST_BUILD_UID="301" NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" -dsclattr() { - /usr/bin/dscl . -read "$1" \ - | awk "/$2/ { print \$2 }" +# caution: may update times on / if not run as normal non-root user +read_only_root() { + # this touch command ~should~ always produce an error + # as of this change I confirmed /usr/bin/touch emits: + # "touch: /: Operation not permitted" Monterey + # "touch: /: Read-only file system" Catalina+ and Big Sur + # "touch: /: Permission denied" Mojave + # (not matching prefix for compat w/ coreutils touch in case using + # an explicit path causes problems; its prefix differs) + case "$(/usr/bin/touch / 2>&1)" in + *"Read-only file system") # Catalina, Big Sur + return 0 + ;; + *"Operation not permitted") # Monterey + return 0 + ;; + *) + return 1 + ;; + esac + + # Avoiding the slow semantic way to get this information (~330ms vs ~8ms) + # unless using touch causes problems. Just in case, that approach is: + # diskutil info -plist / | , i.e. + # diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" - } -poly_validate_assumptions() { - if [ "$(uname -s)" != "Darwin" ]; then - failure "This script is for use with macOS!" +if read_only_root && [ "$NIX_VOLUME_CREATE" = 1 ]; then + should_create_volume() { return 0; } +else + should_create_volume() { return 1; } +fi + +# shellcheck source=./create-darwin-volume.sh +. "$EXTRACTED_NIX_PATH/create-darwin-volume.sh" "no-main" + +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | /usr/bin/awk "/$2/ { print \$2 }" +} + +test_nix_daemon_installed() { + test -e "$NIX_DAEMON_DEST" +} + +poly_cure_artifacts() { + if should_create_volume; then + task "Fixing any leftover Nix volume state" + cat < /dev/null 2>&1 + /usr/sbin/dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 } poly_user_in_group_set() { @@ -151,3 +204,21 @@ poly_create_build_user() { /usr/bin/dscl . create "/Users/$username" \ UniqueID "${uid}" } + +poly_prepare_to_install() { + if should_create_volume; then + header "Preparing a Nix volume" + # intentional indent below to match task indent + cat < 1 )); then + header "Reminders" + for line in "${_reminders[@]}"; do + echo "$line" + if ! headless && [ "${#line}" = 0 ]; then + if read -r -p "Press enter/return to acknowledge."; then + printf $'\033[A\33[2K\r' + fi + fi + done + fi +} + +reminder() { + printf -v label "${BLUE}[ %d ]${ESC}" "$_remind_num" + _reminders+=("$label") + if [[ "$*" = "" ]]; then + while read -r line; do + _reminders+=("$line") + done + else + # this expands each arg to an array entry (and each entry will + # ultimately be a separate line in the output) + _reminders+=("$@") + fi + _reminders+=("") + ((_remind_num++)) +} + __sudo() { local expl="$1" local cmd="$2" @@ -221,18 +314,18 @@ _sudo() { local expl="$1" shift if ! headless; then - __sudo "$expl" "$*" + __sudo "$expl" "$*" >&2 fi sudo "$@" } -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { +readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +finish_cleanup() { rm -rf "$SCRATCH" } -function finish_fail { +finish_fail() { finish_cleanup failure < /dev/null >&2; then warning <&2 -elif [ "$(uname -s)" = "Linux" ]; then +if [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi -INSTALL_MODE=no-daemon -CREATE_DARWIN_VOLUME=0 +case "$(uname -s)" in + "Darwin") + INSTALL_MODE=daemon;; + *) + INSTALL_MODE=no-daemon;; +esac + +# space-separated string +ACTIONS= + # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) - INSTALL_MODE=daemon;; + INSTALL_MODE=daemon + ACTIONS="${ACTIONS}install " + ;; --no-daemon) - INSTALL_MODE=no-daemon;; + if [ "$(uname -s)" = "Darwin" ]; then + printf '\e[1;31mError: --no-daemon installs are no-longer supported on Darwin/macOS!\e[0m\n' >&2 + exit 1 + fi + INSTALL_MODE=no-daemon + # intentional tail space + ACTIONS="${ACTIONS}install " + ;; + # --uninstall) + # # intentional tail space + # ACTIONS="${ACTIONS}uninstall " + # ;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) @@ -69,13 +79,18 @@ while [ $# -gt 0 ]; do --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) - CREATE_DARWIN_VOLUME=1;; + { + echo "Warning: the flag --darwin-use-unencrypted-nix-store-volume" + echo " is no longer needed and will be removed in the future." + echo "" + } >&2;; --nix-extra-conf-file) - export NIX_EXTRA_CONF="$(cat $2)" + # shellcheck disable=SC2155 + export NIX_EXTRA_CONF="$(cat "$2")" shift;; *) - ( - echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" + { + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]" echo "Choose installation method." echo "" @@ -91,55 +106,25 @@ while [ $# -gt 0 ]; do echo "" echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." echo "" - echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" - echo " is installed by default." + echo " --no-modify-profile: Don't modify the user profile to automatically load nix." echo "" echo " --daemon-user-count: Number of build users to create. Defaults to 32." echo "" - echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" + echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix/nix.conf" echo "" if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from." fi - ) >&2 + } >&2 - # darwin and Catalina+ - if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then - ( - echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" - echo " store and mount it at /nix. This is the recommended way to create" - echo " /nix with a read-only / on macOS >=10.15." - echo " See: https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - fi exit;; esac shift done -if [ "$(uname -s)" = "Darwin" ]; then - if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then - printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' - "$self/create-darwin-volume.sh" - fi - - writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)" - if ! [ -e $dest ] && [ "$writable" = "false" ]; then - ( - echo "" - echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - exit 1 - fi -fi - if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" + exec "$self/install-multi-user" $ACTIONS # let ACTIONS split exit 0 fi @@ -149,7 +134,7 @@ fi echo "performing a single-user installation of Nix..." >&2 -if ! [ -e $dest ]; then +if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then @@ -158,12 +143,12 @@ if ! [ -e $dest ]; then fi fi -if ! [ -w $dest ]; then +if ! [ -w "$dest" ]; then echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi -mkdir -p $dest/store +mkdir -p "$dest/store" printf "copying Nix to %s..." "${dest}/store" >&2 # Insert a newline if no progress is shown. @@ -194,6 +179,7 @@ if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then exit 1 fi +# shellcheck source=./nix-profile.sh.in . "$nix/etc/profile.d/nix.sh" if ! "$nix/bin/nix-env" -i "$nix"; then @@ -203,17 +189,17 @@ fi # Install an SSL certificate bundle. if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then - $nix/bin/nix-env -i "$cacert" + "$nix/bin/nix-env" -i "$cacert" export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" fi # Subscribe the user to the Nixpkgs channel and fetch it. if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then - if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + if ! "$nix/bin/nix-channel" --list | grep -q "^nixpkgs "; then + "$nix/bin/nix-channel" --add https://nixos.org/channels/nixpkgs-unstable fi if [ -z "$_NIX_INSTALLER_TEST" ]; then - if ! $nix/bin/nix-channel --update nixpkgs; then + if ! "$nix/bin/nix-channel" --update nixpkgs; then echo "Fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"nix-channel --update nixpkgs\"." fi @@ -229,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" fi added=1 break @@ -240,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" fi added=1 break diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index fda5ef600..f4a2dfc5d 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -15,7 +15,7 @@ readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf create_systemd_override() { header "Configuring proxy for the nix-daemon service" - _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname "$SERVICE_OVERRIDE")" cat < /dev/null 2>&1; then + fetch() { curl -L "$1" -o "$2"; } +elif command -v wget > /dev/null 2>&1; then + fetch() { wget "$1" -O "$2"; } +else + oops "you don't have wget or curl installed, which I need to download the binary tarball" +fi + echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." -curl -L "$url" -o "$tarball" || oops "failed to download '$url'" +fetch "$url" "$tarball" || oops "failed to download '$url'" if command -v sha256sum > /dev/null 2>&1; then hash2="$(sha256sum -b "$tarball" | cut -c1-64)" diff --git a/scripts/local.mk b/scripts/local.mk index 2a0055852..b8477178e 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,7 +1,5 @@ nix_noinst_scripts := \ - $(d)/nix-http-export.cgi \ - $(d)/nix-profile.sh \ - $(d)/nix-reduce-build + $(d)/nix-profile.sh noinst-scripts += $(nix_noinst_scripts) diff --git a/scripts/nix-http-export.cgi.in b/scripts/nix-http-export.cgi.in deleted file mode 100755 index 19a505af1..000000000 --- a/scripts/nix-http-export.cgi.in +++ /dev/null @@ -1,51 +0,0 @@ -#! /bin/sh - -export HOME=/tmp -export NIX_REMOTE=daemon - -TMP_DIR="${TMP_DIR:-/tmp/nix-export}" - -@coreutils@/mkdir -p "$TMP_DIR" || true -@coreutils@/chmod a+r "$TMP_DIR" - -needed_path="?$QUERY_STRING" -needed_path="${needed_path#*[?&]needed_path=}" -needed_path="${needed_path%%&*}" -#needed_path="$(echo $needed_path | ./unhttp)" -needed_path="${needed_path//%2B/+}" -needed_path="${needed_path//%3D/=}" - -echo needed_path: "$needed_path" >&2 - -NIX_STORE="${NIX_STORE_DIR:-/nix/store}" - -echo NIX_STORE: "${NIX_STORE}" >&2 - -full_path="${NIX_STORE}"/"$needed_path" - -if [ "$needed_path" != "${needed_path%.drv}" ]; then - echo "Status: 403 You should create the derivation file yourself" - echo "Content-Type: text/plain" - echo - echo "Refusing to disclose derivation contents" - exit -fi - -if @bindir@/nix-store --check-validity "$full_path"; then - if ! [ -e nix-export/"$needed_path".nar.gz ]; then - @bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz - @coreutils@/ln -fs "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz - fi; - echo "Status: 301 Moved" - echo "Location: nix-export/"$needed_path".nar.gz" - echo -else - echo "Status: 404 No such path found" - echo "Content-Type: text/plain" - echo - echo "Path not found:" - echo "$needed_path" - echo "checked:" - echo "$full_path" -fi - diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 500a98992..0a47571ac 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,7 +5,7 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then +if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt @@ -18,14 +18,14 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS else # Fall back to what is in the nix profiles, favouring whatever is defined last. check_nix_profiles() { - if [ "$ZSH_VERSION" ]; then + if [ -n "$ZSH_VERSION" ]; then # Zsh by default doesn't split words in unquoted parameter expansion. # Set local_options for these options to be reverted at the end of the function # and shwordsplit to force splitting words in $NIX_PROFILES below. setopt local_options shwordsplit fi for i in $NIX_PROFILES; do - if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + if [ -e "$i/etc/ssl/certs/ca-bundle.crt" ]; then export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt fi done diff --git a/scripts/nix-reduce-build.in b/scripts/nix-reduce-build.in deleted file mode 100755 index 50beb9d10..000000000 --- a/scripts/nix-reduce-build.in +++ /dev/null @@ -1,171 +0,0 @@ -#! @bash@ - -WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX); -cd "$WORKING_DIRECTORY"; - -if test -z "$1" || test "a--help" = "a$1" ; then - echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2 - echo As in: >&2 - echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2 - echo nix-reduce-build /etc/nixos/nixos -- \\ - echo " " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2 - echo " store path name will be added into the end of the URL" >&2 - echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2 - echo " that should be a directory where gzipped 'nix-store --export' ">&2 - echo " files are located (they should have .nar.gz extension)" >&2 - echo " Or all together: " >&2 - echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\ - " ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2 - echo " Also supports best-effort local builds of failing expression set:" >&2 - echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2 - echo " nix-daemon:// builds using daemon" - echo " nix-self:// builds directly using nix-store from current installation" >&2 - echo " nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2; - echo "derivations with specified output hash (sha256, sha1 or md5)." >&2 - echo " nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2; - echo "maximum amount of paths" >&2; - echo " nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2; - echo "maximum amount of paths" >&2; - echo " If no package sources are specified, required paths are listed." >&2; - exit; -fi; - -while ! test "$1" = "--" || test "$1" = "" ; do - echo "$1" >> initial; >&2 - shift; -done -shift; -echo Will work on $(cat initial | wc -l) targets. >&2 - -while read ; do - case "$REPLY" in - ${NIX_STORE_DIR:-/nix/store}/*) - echo "$REPLY" >> paths; >&2 - ;; - *) - ( - IFS=: ; - nix-instantiate $REPLY >> paths; - ); - ;; - esac; -done < initial; -echo Proceeding $(cat paths | wc -l) paths. >&2 - -while read; do - case "$REPLY" in - *.drv) - echo "$REPLY" >> derivers; >&2 - ;; - *) - nix-store --query --deriver "$REPLY" >>derivers; - ;; - esac; -done < paths; -echo Found $(cat derivers | wc -l) derivers. >&2 - -cat derivers | xargs nix-store --query -R > derivers-closure; -echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2 - -cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths; -cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths; -echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2 - -cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; -echo We need $(cat needed-paths | wc -l) paths. >&2 - -egrep '[.]drv$' derivers-closure > critical-derivers; - -if test -z "$1" ; then - cat needed-paths; -fi; - -refresh_critical_derivers() { - echo "Finding needed derivers..." >&2; - cat critical-derivers | while read; do - if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then - echo "$REPLY"; - fi; - done > new-critical-derivers; - mv new-critical-derivers critical-derivers; - echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2 -} - -build_here() { - cat critical-derivers | while read; do - echo "Realising $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${REPLY}" - done; -} - -try_to_substitute(){ - cat needed-paths | while read ; do - echo "Building $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}" - done; -} - -for i in "$@"; do - sshHost="${i#ssh://}"; - httpHost="${i#http://}"; - httpsHost="${i#https://}"; - filePath="${i#file:/}"; - if [ "$i" != "$sshHost" ]; then - cat needed-paths | while read; do - echo "Getting $REPLY and its closure over ssh" >&2 - nix-copy-closure --from "$sshHost" --gzip "$REPLY" &2 - curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import; - done; - elif [ "$i" != "$filePath" ] ; then - cat needed-paths | while read; do - echo "Installing $REPLY from file" >&2 - gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import; - done; - elif [ "$i" = "nix-daemon://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self://" ] ; then - NIX_REMOTE= try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE= build_here; - elif [ "$i" = "nix-daemon-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using nix-daemon" >&2 - NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-self-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using direct Nix build" >&2 - NIX_REMOTE= @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-daemon-substitute://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - elif [ "$i" = "nix-self-substitute://" ] ; then - NIX_REMOTE= try_to_substitute; - elif [ "$i" = "nix-daemon-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE= build_here; - fi; - mv needed-paths wanted-paths; - cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; - echo We still need $(cat needed-paths | wc -l) paths. >&2 -done; - -cd / -rm -r "$WORKING_DIRECTORY" diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 92d930384..4b994a753 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -3,7 +3,7 @@ set -e script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo $script | cut -b12-43 -) +installerHash=$(echo "$script" | cut -b12-43 -) installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 57f2cd32d..9d541b45d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -18,6 +18,7 @@ #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" +#include "experimental-features.hh" using namespace nix; using std::cin; @@ -130,11 +131,14 @@ static int main_build_remote(int argc, char * * argv) for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); - if (m.enabled && std::find(m.systemTypes.begin(), - m.systemTypes.end(), - neededSystem) != m.systemTypes.end() && + if (m.enabled + && (neededSystem == "builtin" + || std::find(m.systemTypes.begin(), + m.systemTypes.end(), + neededSystem) != m.systemTypes.end()) && m.allSupported(requiredFeatures) && - m.mandatoryMet(requiredFeatures)) { + m.mandatoryMet(requiredFeatures)) + { rightType = true; AutoCloseFD free; uint64_t load = 0; @@ -270,14 +274,23 @@ connected: { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, ref(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); + copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } uploadLock = -1; auto drv = store->readDerivation(*drvPath); auto outputHashes = staticOutputHashes(*store, drv); - drv.inputSrcs = store->parseStorePathSet(inputs); + + // Hijack the inputs paths of the derivation to include all the paths + // that come from the `inputDrvs` set. + // We don’t do that for the derivations whose `inputDrvs` is empty + // because + // 1. It’s not needed + // 2. Changing the `inputSrcs` set changes the associated output ids, + // which break CA derivations + if (!drv.inputDrvs.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); @@ -286,7 +299,7 @@ connected: std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; @@ -312,13 +325,13 @@ connected: if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ - copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); + copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } // XXX: Should be done as part of `copyPaths` for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); store->registerDrvOutput(realisation); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 2c62bfa7f..4c5d985aa 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -95,8 +95,21 @@ EvalCommand::~EvalCommand() evalState->printStats(); } +ref EvalCommand::getEvalStore() +{ + if (!evalStore) + evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore(); + return ref(evalStore); +} -RealisedPathsCommand::RealisedPathsCommand(bool recursive) +ref EvalCommand::getEvalState() +{ + if (!evalState) + evalState = std::make_shared(searchPath, getEvalStore(), getStore()); + return ref(evalState); +} + +BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { if (recursive) @@ -123,44 +136,53 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive) }); } -void RealisedPathsCommand::run(ref store) +void BuiltPathsCommand::run(ref store) { - std::vector paths; + BuiltPaths paths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(p); + paths.push_back(BuiltPath::Opaque{p}); } else { - auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables); + paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { - auto roots = std::move(pathSet); - pathSet = {}; - RealisedPath::closure(*store, roots, pathSet); + // XXX: This only computes the store path closure, ignoring + // intermediate realisations + StorePathSet pathsRoots, pathsClosure; + for (auto & root : paths) { + auto rootFromThis = root.outPaths(); + pathsRoots.insert(rootFromThis.begin(), rootFromThis.end()); + } + store->computeFSClosure(pathsRoots, pathsClosure); + for (auto & path : pathsClosure) + paths.push_back(BuiltPath::Opaque{path}); } - for (auto & path : pathSet) - paths.push_back(path); } run(store, std::move(paths)); } StorePathsCommand::StorePathsCommand(bool recursive) - : RealisedPathsCommand(recursive) + : BuiltPathsCommand(recursive) { } -void StorePathsCommand::run(ref store, std::vector paths) +void StorePathsCommand::run(ref store, BuiltPaths && paths) { - StorePaths storePaths; - for (auto & p : paths) - storePaths.push_back(p.path()); + StorePathSet storePaths; + for (auto & builtPath : paths) + for (auto & p : builtPath.outPaths()) + storePaths.insert(p); - run(store, std::move(storePaths)); + auto sorted = store->topoSortPaths(storePaths); + std::reverse(sorted.begin(), sorted.end()); + + run(store, std::move(sorted)); } -void StorePathCommand::run(ref store, std::vector storePaths) +void StorePathCommand::run(ref store, std::vector && storePaths) { if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -204,7 +226,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) +void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; @@ -212,22 +234,19 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](const BuiltPath::Opaque & bo) { result.push_back(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](const BuiltPath::Built & bfd) { for (auto & output : bfd.outputs) { - /* Output path should be known because we just tried to - build it. */ - assert(output.second); - result.push_back(*output.second); + result.push_back(output.second); } }, }, buildable.raw()); } if (result.size() != 1) - throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); + throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); updateProfile(result[0]); } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8566e3a6d..0d847d255 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -47,13 +47,18 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; - ref getEvalState(); - EvalCommand(); - std::shared_ptr evalState; - ~EvalCommand(); + + ref getEvalStore(); + + ref getEvalState(); + +private: + std::shared_ptr evalStore; + + std::shared_ptr evalState; }; struct MixFlakeOptions : virtual Args, EvalCommand @@ -105,6 +110,8 @@ enum class Realise { exists. */ Derivation, /* Evaluate in dry-run mode. Postcondition: nothing. */ + // FIXME: currently unused, but could be revived if we can + // evaluate derivations in-memory. Nothing }; @@ -147,7 +154,7 @@ private: }; /* A command that operates on zero or more store paths. */ -struct RealisedPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : public InstallablesCommand { private: @@ -160,26 +167,26 @@ protected: public: - RealisedPathsCommand(bool recursive = false); + BuiltPathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, std::vector paths) = 0; + virtual void run(ref store, BuiltPaths && paths) = 0; void run(ref store) override; bool useDefaultInstallables() override { return !all; } }; -struct StorePathsCommand : public RealisedPathsCommand +struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; - virtual void run(ref store, std::vector storePaths) = 0; + virtual void run(ref store, std::vector && storePaths) = 0; - void run(ref store, std::vector paths) override; + void run(ref store, BuiltPaths && paths) override; }; /* A command that operates on exactly one store path. */ @@ -189,7 +196,7 @@ struct StorePathCommand : public StorePathsCommand virtual void run(ref store, const StorePath & storePath) = 0; - void run(ref store, std::vector storePaths) override; + void run(ref store, std::vector && storePaths) override; }; /* A helper class for registering commands globally. */ @@ -220,26 +227,37 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -DerivedPathsWithHints build(ref store, Realise mode, - std::vector> installables, BuildMode bMode = bmNormal); +BuiltPaths build( + ref evalStore, + ref store, Realise mode, + const std::vector> & installables, + BuildMode bMode = bmNormal); -std::set toStorePaths(ref store, - Realise mode, OperateOn operateOn, - std::vector> installables); - -StorePath toStorePath(ref store, - Realise mode, OperateOn operateOn, - std::shared_ptr installable); - -std::set toDerivations(ref store, - std::vector> installables, - bool useDeriver = false); - -std::set toRealisedPaths( +std::set toStorePaths( + ref evalStore, ref store, Realise mode, OperateOn operateOn, - std::vector> installables); + const std::vector> & installables); + +StorePath toStorePath( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + std::shared_ptr installable); + +std::set toDerivations( + ref store, + const std::vector> & installables, + bool useDeriver = false); + +BuiltPaths toBuiltPaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const std::vector> & installables); /* Helper function to generate args that invoke $EDITOR on filename:lineno. */ @@ -256,7 +274,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const DerivedPathsWithHints & buildables); + void updateProfile(const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 7f7a89b37..5758b52ad 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -58,9 +58,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = "Don't allow lookups in the flake registries.", + .description = + "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", .category = category, - .handler = {&lockFlags.useRegistries, false} + .handler = {[&]() { + lockFlags.useRegistries = false; + warn("'--no-registries' is deprecated; use '--no-use-registries'"); + }} }); addFlag({ @@ -171,14 +175,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { - if (file) return; // FIXME + if (file) { + evalSettings.pureEval = false; + auto state = getEvalState(); + Expr *e = state->parseExprFromFile( + resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) + ); - completeFlakeRefWithFragment( - getEvalState(), - lockFlags, - getDefaultFlakeAttrPathPrefixes(), - getDefaultFlakeAttrPaths(), - prefix); + Value root; + state->eval(e, root); + + auto autoArgs = getAutoArgs(*state); + + std::string prefix_ = std::string(prefix); + auto sep = prefix_.rfind('.'); + std::string searchWord; + if (sep != std::string::npos) { + searchWord = prefix_.substr(sep, std::string::npos); + prefix_ = prefix_.substr(0, sep); + } else { + searchWord = prefix_; + prefix_ = ""; + } + + Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first); + state->forceValue(v1); + Value v2; + state->autoCallFunction(*autoArgs, v1, v2); + + if (v2.type() == nAttrs) { + for (auto & i : *v2.attrs) { + std::string name = i.name; + if (name.find(searchWord) == 0) { + completions->add(i.name); + } + } + } + } else { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); + } } void completeFlakeRefWithFragment( @@ -249,7 +289,6 @@ void completeFlakeRefWithFragment( completeFlakeRef(evalState->store, prefix); } - void completeFlakeRef(ref store, std::string_view prefix) { if (prefix == "") @@ -273,9 +312,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -DerivedPathWithHints Installable::toDerivedPathWithHints() +DerivedPath Installable::toDerivedPath() { - auto buildables = toDerivedPathsWithHints(); + auto buildables = toDerivedPaths(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -309,22 +348,19 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - DerivedPathsWithHints toDerivedPathsWithHints() override + DerivedPaths toDerivedPaths() override { if (storePath.isDerivation()) { - std::map> outputs; auto drv = store->readDerivation(storePath); - for (auto & [name, output] : drv.outputsAndOptPaths(*store)) - outputs.emplace(name, output.second); return { - DerivedPathWithHints::Built { + DerivedPath::Built { .drvPath = storePath, - .outputs = std::move(outputs) + .outputs = drv.outputNames(), } }; } else { return { - DerivedPathWithHints::Opaque { + DerivedPath::Opaque { .path = storePath, } }; @@ -337,22 +373,24 @@ struct InstallableStorePath : Installable } }; -DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() +DerivedPaths InstallableValue::toDerivedPaths() { - DerivedPathsWithHints res; + DerivedPaths res; - std::map>> drvsToOutputs; + std::map> drvsToOutputs; + RealisedPath::Set drvsToCopy; // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { auto outputName = drv.outputName; if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); - drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); + drvsToOutputs[drv.drvPath].insert(outputName); + drvsToCopy.insert(drv.drvPath); } for (auto & i : drvsToOutputs) - res.push_back(DerivedPathWithHints::Built { i.first, i.second }); + res.push_back(DerivedPath::Built { i.first, i.second }); return res; } @@ -564,10 +602,10 @@ InstallableFlake::getCursors(EvalState & state) std::shared_ptr InstallableFlake::getLockedFlake() const { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); - _lockedFlake->flake.config.apply(); - // FIXME: send new config to the daemon. + _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } @@ -616,6 +654,17 @@ std::vector> SourceExprCommand::parseInstallables( for (auto & s : ss) { std::exception_ptr ex; + if (s.find('/') != std::string::npos) { + try { + result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); + continue; + } catch (BadStorePath &) { + } catch (...) { + if (!ex) + ex = std::current_exception(); + } + } + try { auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); result.push_back(std::make_shared( @@ -630,25 +679,7 @@ std::vector> SourceExprCommand::parseInstallables( ex = std::current_exception(); } - if (s.find('/') != std::string::npos) { - try { - result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); - continue; - } catch (BadStorePath &) { - } catch (...) { - if (!ex) - ex = std::current_exception(); - } - } - std::rethrow_exception(ex); - - /* - throw Error( - pathExists(s) - ? "path '%s' is not a flake or a store path" - : "don't know how to handle argument '%s'", s); - */ } } @@ -663,107 +694,121 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -DerivedPathsWithHints build(ref store, Realise mode, - std::vector> installables, BuildMode bMode) +BuiltPaths getBuiltPaths(ref evalStore, ref store, const DerivedPaths & hopefullyBuiltPaths) { - if (mode == Realise::Nothing) - settings.readOnlyMode = true; - - DerivedPathsWithHints buildables; - - std::vector pathsToBuild; - - for (auto & i : installables) { - for (auto & b : i->toDerivedPathsWithHints()) { - std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { - pathsToBuild.push_back(bo); + BuiltPaths res; + for (const auto & b : hopefullyBuiltPaths) + std::visit( + overloaded{ + [&](const DerivedPath::Opaque & bo) { + res.push_back(BuiltPath::Opaque{bo.path}); }, - [&](DerivedPathWithHints::Built bfd) { - StringSet outputNames; - for (auto & output : bfd.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back( - DerivedPath::Built{bfd.drvPath, outputNames}); - }, - }, b.raw()); - buildables.push_back(std::move(b)); - } - } - - if (mode == Realise::Nothing) - printMissing(store, pathsToBuild, lvlError); - else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild, bMode); - - return buildables; -} - -std::set toRealisedPaths( - ref store, - Realise mode, - OperateOn operateOn, - std::vector> installables) -{ - std::set res; - if (operateOn == OperateOn::Output) { - for (auto & b : build(store, mode, installables)) - std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { - res.insert(bo.path); - }, - [&](DerivedPathWithHints::Built bfd) { - auto drv = store->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*store, drv); + [&](const DerivedPath::Built & bfd) { + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*store); for (auto & output : bfd.outputs) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (!outputHashes.count(output.first)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), - output.first); - auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; - auto realisation = store->queryRealisation(outputId); + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled( + Xp::CaDerivations)) { + auto outputId = + DrvOutput{outputHashes.at(output), output}; + auto realisation = + store->queryRealisation(outputId); if (!realisation) - throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string()); - res.insert(RealisedPath{*realisation}); - } - else { - // If ca-derivations isn't enabled, behave as if - // all the paths are opaque to keep the default - // behavior - assert(output.second); - res.insert(*output.second); + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign( + output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); } } + res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); }, - }, b.raw()); - } else { - if (mode == Realise::Nothing) - settings.readOnlyMode = true; - - auto drvPaths = toDerivations(store, installables, true); - res.insert(drvPaths.begin(), drvPaths.end()); - } + }, + b.raw()); return res; } -StorePathSet toStorePaths(ref store, +BuiltPaths build( + ref evalStore, + ref store, + Realise mode, + const std::vector> & installables, + BuildMode bMode) +{ + if (mode == Realise::Nothing) + settings.readOnlyMode = true; + + std::vector pathsToBuild; + + for (auto & i : installables) { + auto b = i->toDerivedPaths(); + pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + } + + if (mode == Realise::Nothing || mode == Realise::Derivation) + printMissing(store, pathsToBuild, lvlError); + else if (mode == Realise::Outputs) + store->buildPaths(pathsToBuild, bMode, evalStore); + + return getBuiltPaths(evalStore, store, pathsToBuild); +} + +BuiltPaths toBuiltPaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const std::vector> & installables) +{ + if (operateOn == OperateOn::Output) + return build(evalStore, store, mode, installables); + else { + if (mode == Realise::Nothing) + settings.readOnlyMode = true; + + BuiltPaths res; + for (auto & drvPath : toDerivations(store, installables, true)) + res.push_back(BuiltPath::Opaque{drvPath}); + return res; + } +} + +StorePathSet toStorePaths( + ref evalStore, + ref store, Realise mode, OperateOn operateOn, - std::vector> installables) + const std::vector> & installables) { StorePathSet outPaths; - for (auto & path : toRealisedPaths(store, mode, operateOn, installables)) - outPaths.insert(path.path()); + for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(thisOutPaths.begin(), thisOutPaths.end()); + } return outPaths; } -StorePath toStorePath(ref store, +StorePath toStorePath( + ref evalStore, + ref store, Realise mode, OperateOn operateOn, std::shared_ptr installable) { - auto paths = toStorePaths(store, mode, operateOn, {installable}); + auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); @@ -771,15 +816,17 @@ StorePath toStorePath(ref store, return *paths.begin(); } -StorePathSet toDerivations(ref store, - std::vector> installables, bool useDeriver) +StorePathSet toDerivations( + ref store, + const std::vector> & installables, + bool useDeriver) { StorePathSet drvPaths; - for (auto & i : installables) - for (auto & b : i->toDerivedPathsWithHints()) + for (const auto & i : installables) + for (const auto & b : i->toDerivedPaths()) std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](const DerivedPath::Opaque & bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -788,7 +835,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](DerivedPathWithHints::Built bfd) { + [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, }, b.raw()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 403403c07..79931ad3e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -23,17 +23,23 @@ struct App // FIXME: add args, sandbox settings, metadata, ... }; +struct UnresolvedApp +{ + App unresolved; + App resolve(ref evalStore, ref store); +}; + struct Installable { virtual ~Installable() { } virtual std::string what() = 0; - virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0; + virtual DerivedPaths toDerivedPaths() = 0; - DerivedPathWithHints toDerivedPathWithHints(); + DerivedPath toDerivedPath(); - App toApp(EvalState & state); + UnresolvedApp toApp(EvalState & state); virtual std::pair toValue(EvalState & state) { @@ -74,7 +80,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - DerivedPathsWithHints toDerivedPathsWithHints() override; + DerivedPaths toDerivedPaths() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index df904612b..1ec258a54 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -8,8 +8,9 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc) libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix -libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown +# libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown +libcmd_LDFLAGS += -llowdown -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers libnix -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index d25113d93..29bb4d31e 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown) struct lowdown_opts opts { .type = LOWDOWN_TERM, .maxdepth = 20, - .cols = std::min(getWindowSize().second, (unsigned short) 80), + .cols = std::max(getWindowSize().second, (unsigned short) 80), .hmargin = 0, .vmargin = 0, .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, @@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown) Finally freeDoc([&]() { lowdown_doc_free(doc); }); size_t maxn = 0; - auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size()); + auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr); if (!node) throw Error("cannot parse Markdown document"); Finally freeNode([&]() { lowdown_node_free(node); }); @@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("cannot allocate Markdown output buffer"); Finally freeBuffer([&]() { lowdown_buf_free(buf); }); - int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node); + int rndr_res = lowdown_term_rndr(buf, renderer, node); if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return std::string(buf->data, buf->size); + return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index bfc131d27..6faa9f9fa 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -68,6 +68,7 @@ struct NixRepl StorePath getDerivationPath(Value & v); bool processLine(string line); void loadFile(const Path & path); + void loadFlake(const std::string & flakeRef); void initEnv(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -104,6 +105,25 @@ NixRepl::~NixRepl() write_history(historyFile.c_str()); } +string runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + + auto res = runProgram(RunOptions { + .program = settings.nixBinDir+ "/" + program, + .args = args, + .environment = subprocessEnv, + .input = input, + }); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + static NixRepl * curRepl; // ugly static char * completionCallback(char * s, int *match) { @@ -180,15 +200,14 @@ namespace { void NixRepl::mainLoop(const std::vector & files) { string error = ANSI_RED "error:" ANSI_NORMAL " "; - std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; + notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); if (!files.empty()) { for (auto & i : files) loadedFiles.push_back(i); - reloadFiles(); - if (!loadedFiles.empty()) std::cout << std::endl; - } + reloadFiles(); + if (!loadedFiles.empty()) notice(""); // Allow nix-repl specific settings in .inputrc rl_readline_name = "nix-repl"; @@ -378,6 +397,8 @@ bool NixRepl::processLine(string line) { if (line == "") return true; + _isInterrupted = false; + string command, arg; if (line[0] == ':') { @@ -397,9 +418,10 @@ bool NixRepl::processLine(string line) << " = Bind expression to variable\n" << " :a Add attributes from resulting set to scope\n" << " :b Build derivation\n" - << " :e Open the derivation in $EDITOR\n" + << " :e Open package or function in $EDITOR\n" << " :i Build derivation, then install result into current profile\n" << " :l Load Nix expression and add it to scope\n" + << " :lf Load Nix flake and add it to scope\n" << " :p Evaluate and print expression recursively\n" << " :q Exit nix-repl\n" << " :r Reload all files\n" @@ -420,6 +442,10 @@ bool NixRepl::processLine(string line) loadFile(arg); } + else if (command == ":lf" || command == ":load-flake") { + loadFlake(arg); + } + else if (command == ":r" || command == ":reload") { state->resetFileCache(); reloadFiles(); @@ -439,14 +465,17 @@ bool NixRepl::processLine(string line) pos = v.lambda.fun->pos; } else { // assume it's a derivation - pos = findDerivationFilename(*state, v, arg); + pos = findPackageFilename(*state, v, arg); } // Open in EDITOR auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, true, args); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); // Reload right after exiting the editor state->resetFileCache(); @@ -456,16 +485,17 @@ bool NixRepl::processLine(string line) else if (command == ":t") { Value v; evalString(arg, v); - std::cout << showType(v) << std::endl; + logger->cout(showType(v)); + } - } else if (command == ":u") { + else if (command == ":u") { Value v, f, result; evalString(arg, v); evalString("drv: (import {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); state->callFunction(f, v, result, Pos()); StorePath drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)}); + runNix("nix-shell", {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { @@ -475,21 +505,15 @@ bool NixRepl::processLine(string line) Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b") { - /* We could do the build in this process using buildPaths(), - but doing it in a child makes it easier to recover from - problems / SIGINT. */ - try { - runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw}); - auto drv = state->store->readDerivation(drvPath); - std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; - for (auto & i : drv.outputsAndOptPaths(*state->store)) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second)); - } catch (ExecError &) { - } + state->store->buildPaths({DerivedPath::Built{drvPath}}); + auto drv = state->store->readDerivation(drvPath); + logger->cout("\nThis derivation produced the following outputs:"); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw}); + runNix("nix-env", {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw}); + runNix("nix-shell", {drvPathRaw}); } } @@ -518,9 +542,9 @@ bool NixRepl::processLine(string line) + concatStringsSep(" ", args) + "\n\n"; } - markdown += trim(stripIndentation(doc->doc)); + markdown += stripIndentation(doc->doc); - std::cout << renderMarkdownToTerminal(markdown); + logger->cout(trim(renderMarkdownToTerminal(markdown))); } else throw Error("value does not have documentation"); } @@ -561,6 +585,25 @@ void NixRepl::loadFile(const Path & path) addAttrsToScope(v2); } +void NixRepl::loadFlake(const std::string & flakeRefS) +{ + auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) + throw Error("cannot use ':load-flake' on mutable flake reference '%s' (use --impure to override)", flakeRefS); + + Value v; + + flake::callFlake(*state, + flake::lockFlake(*state, flakeRef, + flake::LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); + addAttrsToScope(v); +} + void NixRepl::initEnv() { @@ -584,9 +627,9 @@ void NixRepl::reloadFiles() bool first = true; for (auto & i : old) { - if (!first) std::cout << std::endl; + if (!first) notice(""); first = false; - std::cout << format("Loading '%1%'...") % i << std::endl; + notice("Loading '%1%'...", i); loadFile(i); } } @@ -596,8 +639,8 @@ void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs); for (auto & i : *attrs.attrs) - addVarToScope(i.name, i.value); - std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; + addVarToScope(i.name, *i.value); + notice("Added %1% variables.", attrs.attrs->size()); } @@ -605,8 +648,9 @@ void NixRepl::addVarToScope(const Symbol & name, Value * v) { if (displ >= envSize) throw Error("environment full; cannot add more variables"); - staticEnv->vars[name] = displ; - env->values[displ++] = v; + staticEnv->vars.emplace_back(name, displ); + staticEnv->sort(); + env->values[displ++] = &v; varNames.insert((string) name); } @@ -665,7 +709,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nString: - str << ANSI_YELLOW; + str << ANSI_WARNING; printStringValue(str, v.string.s); str << ANSI_NORMAL; break; diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9dd557205..c50c6d92b 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s) ++i; while (1) { if (i == s.end()) - throw Error("missing closing quote in selection path '%1%'", s); + throw ParseError("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -100,7 +100,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr } -Pos findDerivationFilename(EvalState & state, Value & v, std::string what) +Pos findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto colon = pos.rfind(':'); if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); + throw ParseError("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); unsigned int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); + throw ParseError("cannot parse line number '%s'", pos); } Symbol file = state.symbols.create(filename); diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index d9d74ab2d..2ee3ea089 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -14,7 +14,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr Bindings & autoArgs, Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ -Pos findDerivationFilename(EvalState & state, Value & v, std::string what); +Pos findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 1da8d91df..7d6ffc9f3 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -17,8 +17,8 @@ struct Attr { Symbol name; Value * value; - Pos * pos; - Attr(Symbol name, Value * value, Pos * pos = &noPos) + ptr pos; + Attr(Symbol name, Value * value, ptr pos = ptr(&noPos)) : name(name), value(value), pos(pos) { }; Attr() : pos(&noPos) { }; bool operator < (const Attr & a) const @@ -35,13 +35,13 @@ class Bindings { public: typedef uint32_t size_t; - Pos *pos; + ptr pos; private: size_t size_, capacity_; Attr attrs[0]; - Bindings(size_t capacity) : size_(0), capacity_(capacity) { } + Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { } Bindings(const Bindings & bindings) = delete; public: diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index aa14bf79b..fb0932c00 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -61,6 +61,14 @@ MixEvalArgs::MixEvalArgs() fetchers::overrideRegistry(from.input, to.input, extraAttrs); }} }); + + addFlag({ + .longName = "eval-store", + .description = "The Nix store to use for evaluations.", + .category = category, + .labels = {"store-url"}, + .handler = {&evalStoreUrl}, + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index be7fda783..0e113fff1 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args Strings searchPath; -private: + std::optional evalStoreUrl; +private: std::map autoArgs; }; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 11a61da26..a20123f34 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -65,7 +65,11 @@ static char * dupStringWithLen(const char * s, size_t size) RootValue allocRootValue(Value * v) { +#if HAVE_BOEHMGC return std::allocate_shared(traceable_allocator(), v); +#else + return std::make_shared(v); +#endif } @@ -234,22 +238,34 @@ static void * oomHandler(size_t requested) } class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) }; + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { + return sctx.size - boost::context::stack_traits::page_size(); + } + public: boost::context::stack_context allocate() override { auto sctx = stack.allocate(); - GC_add_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); return sctx; } void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - sctx.size, sctx.sp); + GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); stack.deallocate(sctx); } @@ -363,7 +379,10 @@ static Strings parseNixPath(const string & s) } -EvalState::EvalState(const Strings & _searchPath, ref store) +EvalState::EvalState( + const Strings & _searchPath, + ref store, + std::shared_ptr buildStore) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -396,6 +415,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , buildStore(buildStore ? buildStore : store) , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(new StaticEnv(false, 0)) @@ -426,12 +446,12 @@ EvalState::EvalState(const Strings & _searchPath, ref store) StorePathSet closure; store->computeFSClosure(store->toStorePath(r.second).first, closure); for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + allowPath(path); } catch (InvalidPath &) { - allowedPaths->insert(r.second); + allowPath(r.second); } } else - allowedPaths->insert(r.second); + allowPath(r.second); } } @@ -446,6 +466,35 @@ EvalState::~EvalState() } +void EvalState::requireExperimentalFeatureOnEvaluation( + const ExperimentalFeature & feature, + const std::string_view fName, + const Pos & pos) +{ + if (!settings.isExperimentalFeatureEnabled(feature)) { + throw EvalError({ + .msg = hintfmt( + "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.", + feature, + fName + ), + .errPos = pos + }); + } +} + +void EvalState::allowPath(const Path & path) +{ + if (allowedPaths) + allowedPaths->insert(path); +} + +void EvalState::allowPath(const StorePath & storePath) +{ + if (allowedPaths) + allowedPaths->insert(store->toRealPath(storePath)); +} + Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; @@ -472,7 +521,7 @@ Path EvalState::checkSourcePath(const Path & path_) } if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + throw RestrictedPathError("access to absolute path '%1%' is forbidden in restricted mode", abspath); /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); @@ -485,7 +534,7 @@ Path EvalState::checkSourcePath(const Path & path_) } } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); + throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); } @@ -535,14 +584,20 @@ Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv->vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addConstant(name, v2); return v2; } +void EvalState::addConstant(const string & name, Value * v) +{ + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); +} + + Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { @@ -561,7 +616,7 @@ Value * EvalState::addPrimOp(const string & name, Value * v = allocValue(); v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv->vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); return v; @@ -587,7 +642,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value * v = allocValue(); v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv->vars[envName] = baseEnvDispl; + staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); return v; @@ -861,7 +916,7 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (size_t l = var.level; l; --l, env = env->up) ; + for (auto l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; @@ -875,7 +930,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { - if (countCalls && j->pos) attrSelects[*j->pos]++; + if (countCalls) attrSelects[*j->pos]++; return j->value; } if (!env->prevWith) { @@ -886,18 +941,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } -std::atomic nrValuesFreed{0}; - -void finalizeValue(void * obj, void * data) -{ - nrValuesFreed++; -} - Value * EvalState::allocValue() { nrValues++; auto v = (Value *) allocBytes(sizeof(Value)); - //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); return v; } @@ -949,9 +996,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr) } -void EvalState::mkPos(Value & v, Pos * pos) +void EvalState::mkPos(Value & v, ptr pos) { - if (pos && pos->file.set()) { + if (pos->file.set()) { mkAttrs(v, 3); mkString(*allocAttr(v, sFile), pos->file); mkInt(*allocAttr(v, sLine), pos->line); @@ -974,39 +1021,37 @@ Value * Expr::maybeThunk(EvalState & state, Env & env) } -unsigned long nrAvoided = 0; - Value * ExprVar::maybeThunk(EvalState & state, Env & env) { Value * v = state.lookupVar(&env, *this, true); /* The value might not be initialised in the environment yet. In that case, ignore it. */ - if (v) { nrAvoided++; return v; } + if (v) { state.nrAvoided++; return v; } return Expr::maybeThunk(state, env); } Value * ExprString::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprInt::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprFloat::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprPath::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } @@ -1021,38 +1066,23 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) return; } - Path path2 = resolveExprPath(path); - if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + Path resolvedPath = resolveExprPath(path); + if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; } - printTalkative("evaluating file '%1%'", path2); + printTalkative("evaluating file '%1%'", resolvedPath); Expr * e = nullptr; - auto j = fileParseCache.find(path2); + auto j = fileParseCache.find(resolvedPath); if (j != fileParseCache.end()) e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(path2)); + e = parseExprFromFile(checkSourcePath(resolvedPath)); - fileParseCache[path2] = e; - - try { - // Enforce that 'flake.nix' is a direct attrset, not a - // computation. - if (mustBeTrivial && - !(dynamic_cast(e))) - throw Error("file '%s' must be an attribute set", path); - eval(e, v); - } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", path2); - throw; - } - - fileEvalCache[path2] = v; - if (path != path2) fileEvalCache[path] = v; + cacheFile(path, resolvedPath, e, v, mustBeTrivial); } @@ -1063,6 +1093,32 @@ void EvalState::resetFileCache() } +void EvalState::cacheFile( + const Path & path, + const Path & resolvedPath, + Expr * e, + Value & v, + bool mustBeTrivial) +{ + fileParseCache[resolvedPath] = e; + + try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast(e))) + throw EvalError("file '%s' must be an attribute set", path); + eval(e, v); + } catch (Error & e) { + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); + throw; + } + + fileEvalCache[resolvedPath] = v; + if (path != resolvedPath) fileEvalCache[path] = v; +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -1144,7 +1200,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -1153,7 +1209,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) } else vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); + v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos))); } /* If the rec contains an attribute called `__overrides', then @@ -1185,7 +1241,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) else for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos))); /* Dynamic attrs apply *after* rec and __overrides. */ for (auto & i : dynamicAttrs) { @@ -1203,11 +1259,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos))); v.attrs->sort(); // FIXME: inefficient } - v.attrs->pos = &pos; + v.attrs->pos = ptr(&pos); } @@ -1221,7 +1277,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -1262,12 +1318,10 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa } -unsigned long nrLookups = 0; - void ExprSelect::eval(EvalState & state, Env & env, Value & v) { Value vTmp; - Pos * pos2 = 0; + ptr pos2(&noPos); Value * vAttrs = &vTmp; e->eval(state, env, vTmp); @@ -1275,7 +1329,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) try { for (auto & i : attrPath) { - nrLookups++; + state.nrLookups++; Bindings::iterator j; Symbol name = getName(i, state, env); if (def) { @@ -1293,13 +1347,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } vAttrs = j->value; pos2 = j->pos; - if (state.countCalls && pos2) state.attrSelects[*pos2]++; + if (state.countCalls) state.attrSelects[*pos2]++; } - state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); + state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) ); } catch (Error & e) { - if (pos2 && pos2->file != state.sDerivationNix) + if (*pos2 != noPos && pos2->file != state.sDerivationNix) addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); throw; @@ -1340,160 +1394,183 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } -void ExprApp::eval(EvalState & state, Env & env, Value & v) -{ - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} - - -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->isPrimOpApp()) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.mkPrimOpApp(fun2, &arg); - } -} - - -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) { auto trace = evalSettings.traceFunctionCalls ? std::make_unique(pos) : nullptr; forceValue(fun, pos); - if (fun.isPrimOp() || fun.isPrimOpApp()) { - callPrimOp(fun, arg, v, pos); - return; - } + Value vCur(fun); - if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); - } - } + auto makeAppChain = [&]() + { + vRes = vCur; + for (size_t i = 0; i < nrArgs; ++i) { + auto fun2 = allocValue(); + *fun2 = vRes; + vRes.mkPrimOpApp(fun2, args[i]); + } + }; - if (!fun.isLambda()) { - throwTypeError( - pos, - "attempt to call something which is not a function but %1%", - showType(fun).c_str(), - fakeEnv(1), 0); - } + Attr * functor; - ExprLambda & lambda(*fun.lambda.fun); + while (nrArgs > 0) { - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; + if (vCur.isLambda()) { - size_t displ = 0; + ExprLambda & lambda(*vCur.lambda.fun); - if (!lambda.matchAttrs){ - env2.values[displ++] = &arg; - } - else { - forceAttrs(arg, pos); + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = vCur.lambda.env; - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; + Displacement displ = 0; - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) - throwTypeError( - pos, - "%1% called without required argument '%2%'", - lambda, - i.name, - *fun.lambda.env, &lambda); - env2.values[displ++] = i.def->maybeThunk(*this, env2); + if (!lambda.hasFormals()) + env2.values[displ++] = args[0]; + else { + forceAttrs(*args[0], pos); + + if (!lambda.arg.empty()) + env2.values[displ++] = args[0]; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + auto j = args[0]->attrs->get(i.name); + if (!j) { + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name, *fun.lambda.env, &lambda); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *args[0]->attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. */ + try { + lambda.body->eval(*this, env2, vCur); + } catch (Error & e) { + if (loggerSettings.showTrace.get()) { + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); + addErrorTrace(e, pos, "from call site%s", ""); + } + throw; + } + + nrArgs--; + args += 1; + } + + else if (vCur.isPrimOp()) { + + size_t argsLeft = vCur.primOp->arity; + + if (nrArgs < argsLeft) { + /* We don't have enough arguments, so create a tPrimOpApp chain. */ + makeAppChain(); + return; } else { - attrsUsed++; - env2.values[displ++] = j->value; + /* We have all the arguments, so call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[vCur.primOp->name]++; + vCur.primOp->fun(*this, pos, args, vCur); + + nrArgs -= argsLeft; + args += argsLeft; } } + else if (vCur.isPrimOpApp()) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &vCur; + while (primOp->isPrimOpApp()) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->isPrimOp()); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError(pos, - "%1% called with unexpected argument '%2%'", - lambda, - i.name, - *fun.lambda.env, &lambda); - abort(); // can't happen + if (nrArgs < argsLeft) { + /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ + makeAppChain(); + return; + } else { + /* We have all the arguments, so call the primop with + the previous and new arguments. */ + + Value * vArgs[arity]; + auto n = argsDone; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) + vArgs[--n] = arg->primOpApp.right; + + for (size_t i = 0; i < argsLeft; ++i) + vArgs[argsDone + i] = args[i]; + + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, vCur); + + nrArgs -= argsLeft; + args += argsLeft; + } } + + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur' may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue(), args[0]}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; + } + + else + throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); } - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); + vRes = vCur; +} - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (loggerSettings.showTrace.get()) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + +void ExprCall::eval(EvalState & state, Env & env, Value & v) +{ + Value vFun; + fun->eval(state, env, vFun); + + Value * vArgs[args.size()]; + for (size_t i = 0; i < args.size(); ++i) + vArgs[i] = args[i]->maybeThunk(state, env); + + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -1519,7 +1596,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { res = fun; return; } @@ -1722,7 +1799,6 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) and none of the strings are allowed to have contexts. */ if (first) { firstType = vTmp.type(); - first = false; } if (firstType == nInt) { @@ -1744,7 +1820,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp), env, this); } else - s << state.coerceToString(pos, vTmp, context, false, firstType == nString); + /* skip canonization of first path, which would only be not + canonized in the first place if it's coming from a ./${foo} type + path */ + s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first); + + first = false; } if (firstType == nInt) @@ -1763,7 +1844,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) void ExprPos::eval(EvalState & state, Env & env, Value & v) { - state.mkPos(v, &pos); + state.mkPos(v, ptr(&pos)); } @@ -1935,7 +2016,7 @@ std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, } string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore) + bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); @@ -1947,7 +2028,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } if (v.type() == nPath) { - Path path(canonPath(v.path)); + Path path(canonicalizePath ? canonPath(v.path) : v.path); return copyToStore ? copyPathToStore(context, path) : path; } @@ -2010,6 +2091,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); + allowPath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 91e43ddfe..485c2df83 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,6 +5,7 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" +#include "experimental-features.hh" #include #include @@ -98,8 +99,14 @@ public: Value vEmptySet; + /* Store used to materialise .drv files. */ const ref store; + /* Store used to build stuff. */ + const ref buildStore; + + RootValue vCallFlake = nullptr; + RootValue vImportedDrvToDerivation = nullptr; private: SrcToStore srcToStore; @@ -132,13 +139,31 @@ private: public: - EvalState(const Strings & _searchPath, ref store); + EvalState( + const Strings & _searchPath, + ref store, + std::shared_ptr buildStore = nullptr); ~EvalState(); + void requireExperimentalFeatureOnEvaluation( + const ExperimentalFeature &, + const std::string_view fName, + const Pos & pos + ); + void addToSearchPath(const string & s); SearchPath getSearchPath() { return searchPath; } + /* Allow access to a path. */ + void allowPath(const Path & path); + + /* Allow access to a store path. Note that this gets remapped to + the real store path if `store` is a chroot store. */ + void allowPath(const StorePath & storePath); + + /* Check whether access to a path is allowed and throw an error if + not. Otherwise return the canonicalised path. */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); @@ -167,6 +192,14 @@ public: trivial (i.e. doesn't require arbitrary computation). */ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); + /* Like `cacheFile`, but with an already parsed expression. */ + void cacheFile( + const Path & path, + const Path & resolvedPath, + Expr * e, + Value & v, + bool mustBeTrivial = false); + void resetFileCache(); /* Look up a file in the search path. */ @@ -221,7 +254,8 @@ public: booleans and lists to a string. If `copyToStore' is set, referenced paths are copied to the Nix store as a side effect. */ string coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore = false, bool copyToStore = true); + bool coerceMore = false, bool copyToStore = true, + bool canonicalizePath = true); string copyPathToStore(PathSet & context, const Path & path); @@ -247,6 +281,8 @@ private: Value * addConstant(const string & name, Value & v); + void addConstant(const string & name, Value * v); + Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); @@ -286,8 +322,14 @@ public: bool isFunctor(Value & fun); - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + // FIXME: use std::span + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + + void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + { + Value * args[] = {&arg}; + callFunction(fun, 1, args, vRes, pos); + } /* Automatically call a function for which each argument has a default value or has a binding in the `args' map. */ @@ -305,7 +347,7 @@ public: void mkList(Value & v, size_t length); void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); - void mkPos(Value & v, Pos * pos); + void mkPos(Value & v, ptr pos); void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); @@ -320,8 +362,10 @@ private: unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; unsigned long nrListElems = 0; + unsigned long nrLookups = 0; unsigned long nrAttrsets = 0; unsigned long nrAttrsInAttrsets = 0; + unsigned long nrAvoided = 0; unsigned long nrOpUpdates = 0; unsigned long nrOpUpdateValuesCopied = 0; unsigned long nrListConcats = 0; @@ -343,6 +387,11 @@ private: friend struct ExprOpUpdate; friend struct ExprOpConcatLists; + friend struct ExprVar; + friend struct ExprString; + friend struct ExprInt; + friend struct ExprFloat; + friend struct ExprPath; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 63566131e..c03f4106c 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "globals.hh" #include @@ -22,12 +23,14 @@ static TrustedList readTrustedList() static void writeTrustedList(const TrustedList & trustedList) { - writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); + auto path = trustedListPath(); + createDirs(dirOf(path)); + writeFile(path, nlohmann::json(trustedList).dump()); } void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-suffix"}; + std::set whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"}; for (auto & [name, value] : settings) { @@ -50,21 +53,19 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + if (nix::settings.acceptFlakeConfig){ + trusted = true; + } else if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { trusted = *saved; + warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { + trusted = true; + } + if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') { + trustedList[name][valueS] = trusted; + writeTrustedList(trustedList); } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2e94490d4..f5be67d67 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "eval.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" @@ -63,8 +64,7 @@ static std::tuple fetchOrSubstituteTree( debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -88,10 +88,12 @@ static void expectType(EvalState & state, ValueType type, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const Pos & pos); + EvalState & state, Value * value, const Pos & pos, + const std::optional & baseDir); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const Pos & pos) + const std::string & inputName, Value * value, const Pos & pos, + const std::optional & baseDir) { expectType(state, nAttrs, *value, pos); @@ -115,7 +117,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); @@ -153,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); if (url) - input.ref = parseFlakeRef(*url, {}, true); + input.ref = parseFlakeRef(*url, baseDir, true); } if (!input.follows && !input.ref) @@ -163,7 +165,8 @@ static FlakeInput parseFlakeInput(EvalState & state, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const Pos & pos) + EvalState & state, Value * value, const Pos & pos, + const std::optional & baseDir) { std::map inputs; @@ -174,7 +177,8 @@ static std::map parseFlakeInputs( parseFlakeInput(state, inputAttr.name, inputAttr.value, - *inputAttr.pos)); + *inputAttr.pos, + baseDir)); } return inputs; @@ -190,7 +194,8 @@ static Flake getFlake( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir); + auto flakeFile = canonPath(flakeDir + "/flake.nix"); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", lockedRef, state.store->printStorePath(sourceInfo.storePath)); @@ -218,14 +223,14 @@ static Flake getFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir); auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { @@ -292,11 +297,18 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature("flakes"); + settings.requireExperimentalFeature(Xp::Flakes); FlakeCache flakeCache; - auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + + auto flake = getFlake(state, topRef, useRegistries, flakeCache); + + if (lockFlags.applyNixConfig) { + flake.config.apply(); + state.store->setOptions(); + } try { @@ -317,25 +329,38 @@ LockedFlake lockFlake( std::vector parents; + struct LockParent { + /* The path to this parent. */ + InputPath path; + + /* Whether we are currently inside a top-level lockfile + (inputs absolute) or subordinate lockfile (inputs + relative). */ + bool absolute; + }; + std::function node, const InputPath & inputPathPrefix, - std::shared_ptr oldNode)> + std::shared_ptr oldNode, + const LockParent & parent, + const Path & parentPath)> computeLocks; computeLocks = [&]( const FlakeInputs & flakeInputs, std::shared_ptr node, const InputPath & inputPathPrefix, - std::shared_ptr oldNode) + std::shared_ptr oldNode, + const LockParent & parent, + const Path & parentPath) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - // FIXME: check this - for (auto & [id, input] : flake.inputs) { + for (auto & [id, input] : flakeInputs) { for (auto & [idOverride, inputOverride] : input.overrides) { auto inputPath(inputPathPrefix); inputPath.push_back(id); @@ -359,22 +384,31 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); - if (hasOverride) overridesUsed.insert(inputPath); + if (hasOverride) { + overridesUsed.insert(inputPath); + // Respect the “flakeness” of the input even if we + // override it + i->second.isFlake = input2.isFlake; + } auto & input = hasOverride ? i->second : input2; /* Resolve 'follows' later (since it may refer to an input path we haven't processed yet. */ if (input.follows) { InputPath target; - if (hasOverride || input.absolute) - /* 'follows' from an override is relative to the - root of the graph. */ + + if (parent.absolute && !hasOverride) { target = *input.follows; - else { - /* Otherwise, it's relative to the current flake. */ - target = inputPathPrefix; + } else { + if (hasOverride) { + target = inputPathPrefix; + target.pop_back(); + } else + target = parent.path; + for (auto & i : *input.follows) target.push_back(i); } + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); node->inputs.insert_or_assign(id, target); continue; @@ -412,22 +446,18 @@ LockedFlake lockFlake( update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); - } else { + FlakeInputs fakeInputs; + + if (!mustRefetch) { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ - FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -435,16 +465,28 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { + auto o = input.overrides.find(i.first); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == input.overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, - .absolute = true }); } } - - computeLocks(fakeInputs, childNode, inputPath, oldLock); } + computeLocks( + mustRefetch + ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs + : fakeInputs, + childNode, inputPath, oldLock, parent, parentPath); + } else { /* We need to create a new lock file entry. So fetch this input. */ @@ -454,7 +496,15 @@ LockedFlake lockFlake( throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { - auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); + Path localPath = parentPath; + FlakeRef localRef = *input.ref; + + // If this input is a path, recurse it down. + // This allows us to resolve path inputs relative to the current flake. + if (localRef.input.getType() == "path") + localPath = absPath(*input.ref->input.getSourcePath(), parentPath); + + auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -475,6 +525,13 @@ LockedFlake lockFlake( parents.push_back(*input.ref); Finally cleanup([&]() { parents.pop_back(); }); + // Follows paths from existing inputs in the top-level lockfile are absolute, + // whereas paths in subordinate lockfiles are relative to those lockfiles. + LockParent newParent { + .path = inputPath, + .absolute = oldLock ? true : false + }; + /* Recursively process the inputs of this flake. Also, unless we already have this flake in the top-level lock file, use this flake's @@ -484,12 +541,13 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, + newParent, localPath); } else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, lockFlags.useRegistries, flakeCache); + state, *input.ref, useRegistries, flakeCache); node->inputs.insert_or_assign(id, std::make_shared(lockedRef, *input.ref, false)); } @@ -502,9 +560,17 @@ LockedFlake lockFlake( } }; + LockParent parent { + .path = {}, + .absolute = true + }; + + // Bring in the current ref for relative path resolution if we have it + auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir); + computeLocks( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -554,8 +620,8 @@ LockedFlake lockFlake( topRef.input.markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", lockFlags.commitLockFile - ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", - relPath, lockFileExists ? "Update" : "Add", diff)) + ? std::optional(fmt("%s: %s\n\nFlake lock file changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true))) : std::nullopt); /* Rewriting the lockfile changed the top-level @@ -563,7 +629,7 @@ LockedFlake lockFlake( also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; FlakeCache dummyCache; - flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); + flake = getFlake(state, topRef, useRegistries, dummyCache); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && @@ -580,8 +646,10 @@ LockedFlake lockFlake( } } else throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); - } else + } else { warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + flake.forceDirty = true; + } } return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; @@ -604,26 +672,32 @@ void callFlake(EvalState & state, mkString(*vLocks, lockedFlake.lockFile.to_string()); - emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + emitTreeAttrs( + state, + *lockedFlake.flake.sourceInfo, + lockedFlake.flake.lockedRef.input, + *vRootSrc, + false, + lockedFlake.flake.forceDirty); mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); - static RootValue vCallFlake = nullptr; - - if (!vCallFlake) { - vCallFlake = allocRootValue(state.allocValue()); + if (!state.vCallFlake) { + state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , "/"), **vCallFlake); + , "/"), **state.vCallFlake); } - state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos); + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); if (evalSettings.pureEval && !flakeRef.input.isImmutable()) @@ -633,13 +707,13 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, - .useRegistries = !evalSettings.pureEval, + .useRegistries = !evalSettings.pureEval && settings.useRegistries, .allowMutable = !evalSettings.pureEval, }), v); } -static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes"); +static RegisterPrimOp r2("__getFlake", 1, prim_getFlake); } @@ -649,8 +723,9 @@ Fingerprint LockedFlake::getFingerprint() const // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, - fmt("%s;%d;%d;%s", + fmt("%s;%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), + flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), lockFile)); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index d17d5e183..524b18af1 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -43,7 +43,6 @@ struct FlakeInput std::optional ref; bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path std::optional follows; - bool absolute = false; // whether 'follows' is relative to the flake root FlakeInputs overrides; }; @@ -59,9 +58,10 @@ struct ConfigFile /* The contents of a flake.nix file. */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher + FlakeRef originalRef; // the original flake specification (by the user) + FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake + FlakeRef lockedRef; // the specific local store result of invoking the fetcher + bool forceDirty = false; // pretend that 'lockedRef' is dirty std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; @@ -102,7 +102,11 @@ struct LockFlags /* Whether to use the registries to lookup indirect flake references like 'nixpkgs'. */ - bool useRegistries = true; + std::optional useRegistries = std::nullopt; + + /* Whether to apply flake's nixConfig attribute to the configuration */ + + bool applyNixConfig = false; /* Whether mutable flake references (i.e. those without a Git revision or similar) without a corresponding lock are @@ -137,6 +141,8 @@ void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, const fetchers::Input & input, - Value & v, bool emptyRevFallback = false); + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); } diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 833e8a776..29128d789 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -172,8 +172,12 @@ std::pair parseFlakeRefWithFragment( auto parsedURL = parseURL(url); std::string fragment; std::swap(fragment, parsedURL.fragment); + + auto input = Input::fromURL(parsedURL); + input.parent = baseDir; + return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")), fragment); } } diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 6089d1363..fda340789 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,6 +2,8 @@ #include "store-api.hh" #include "url-parts.hh" +#include + #include namespace nix::flake { @@ -268,10 +270,20 @@ std::map LockFile::getAllInputs() const return res; } +static std::string describe(const FlakeRef & flakeRef) +{ + auto s = fmt("'%s'", flakeRef.to_string()); + + if (auto lastModified = flakeRef.input.getLastModified()) + s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); + + return s; +} + std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge) { if (auto node = std::get_if<0>(&edge)) - stream << "'" << (*node)->lockedRef << "'"; + stream << describe((*node)->lockedRef); else if (auto follows = std::get_if<1>(&edge)) stream << fmt("follows '%s'", printInputPath(*follows)); return stream; @@ -299,14 +311,15 @@ std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) while (i != oldFlat.end() || j != newFlat.end()) { if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { - res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second); + res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", + printInputPath(j->first), j->second); ++j; } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { - res += fmt("* Removed '%s'\n", printInputPath(i->first)); + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first)); ++i; } else { if (!equals(i->second, j->second)) { - res += fmt("* Updated '%s': %s -> %s\n", + res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", printInputPath(i->first), i->second, j->second); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 7298419d9..c18877e29 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -9,6 +9,9 @@ %s DEFAULT %x STRING %x IND_STRING +%x INPATH +%x INPATH_SLASH +%x PATH_START %{ @@ -25,6 +28,8 @@ using namespace nix; namespace nix { +// backup to recover from yyless(0) +YYLTYPE prev_yylloc; static void initLoc(YYLTYPE * loc) { @@ -35,14 +40,18 @@ static void initLoc(YYLTYPE * loc) static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) { + prev_yylloc = *loc; + loc->first_line = loc->last_line; loc->first_column = loc->last_column; - while (len--) { + for (size_t i = 0; i < len; i++) { switch (*s++) { case '\r': - if (*s == '\n') /* cr/lf */ + if (*s == '\n') { /* cr/lf */ + i++; s++; + } /* fall through */ case '\n': ++loc->last_line; @@ -55,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } +// FIXME: optimize static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; @@ -95,9 +105,12 @@ ANY .|\n ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? -PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? -HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? -SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +PATH_CHAR [a-zA-Z0-9\.\_\-\+] +PATH {PATH_CHAR}*(\/{PATH_CHAR}+)+\/? +PATH_SEG {PATH_CHAR}*\/ +HPATH \~(\/{PATH_CHAR}+)+\/? +HPATH_START \~\/ +SPATH \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\> URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ @@ -198,17 +211,75 @@ or { return OR_KW; } return IND_STR; } +{PATH_SEG}\$\{ | +{HPATH_START}\$\{ { + PUSH_STATE(PATH_START); + yyless(0); + *yylloc = prev_yylloc; +} + +{PATH_SEG} { + POP_STATE(); + PUSH_STATE(INPATH_SLASH); + yylval->path = strdup(yytext); + return PATH; +} + +{HPATH_START} { + POP_STATE(); + PUSH_STATE(INPATH_SLASH); + yylval->path = strdup(yytext); + return HPATH; +} + +{PATH} { + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->path = strdup(yytext); + return PATH; +} +{HPATH} { + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->path = strdup(yytext); + return HPATH; +} + +\$\{ { + POP_STATE(); + PUSH_STATE(INPATH); + PUSH_STATE(DEFAULT); + return DOLLAR_CURLY; +} +{PATH}|{PATH_SEG}|{PATH_CHAR}+ { + POP_STATE(); + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->e = new ExprString(data->symbols.create(string(yytext))); + return STR; +} +{ANY} | +<> { + /* if we encounter a non-path character we inform the parser that the path has + ended with a PATH_END token and re-parse this character in the default + context (it may be ')', ';', or something of that sort) */ + POP_STATE(); + yyless(0); + *yylloc = prev_yylloc; + return PATH_END; +} + +{ANY} | +<> { + throw ParseError("path has a trailing slash"); +} -{PATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path '%s' has a trailing slash", yytext); - yylval->path = strdup(yytext); - return PATH; - } -{HPATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path '%s' has a trailing slash", yytext); - yylval->path = strdup(yytext); - return HPATH; - } {SPATH} { yylval->path = strdup(yytext); return SPATH; } {URI} { yylval->uri = strdup(yytext); return URI; } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 26c53d301..016631647 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -15,8 +15,8 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS = -lboost_context -ifneq ($(OS), FreeBSD) +libexpr_LDFLAGS += -lboost_context -pthread +ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif @@ -35,7 +35,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 3e42789a2..696b149e3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -123,7 +123,7 @@ void ExprList::show(std::ostream & str) const void ExprLambda::show(std::ostream & str) const { str << "("; - if (matchAttrs) { + if (hasFormals()) { str << "{ "; bool first = true; for (auto & i : formals->formals) { @@ -142,6 +142,16 @@ void ExprLambda::show(std::ostream & str) const str << ": " << *body << ")"; } +void ExprCall::show(std::ostream & str) const +{ + str << '(' << *fun; + for (auto e : args) { + str << ' '; + str << *e; + } + str << ')'; +} + void ExprLet::show(std::ostream & str) const { str << "(let "; @@ -273,13 +283,13 @@ void ExprVar::bindVars(const std::shared_ptr &env) /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; - unsigned int level; + Level level; int withLevel = -1; for (curEnv = env.get(), level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + auto i = curEnv->find(name); if (i != curEnv->vars.end()) { fromWith = false; this->level = level; @@ -332,12 +342,13 @@ void ExprAttrs::bindVars(const std::shared_ptr &env) staticenv = env; if (recursive) { - auto newEnv = std::shared_ptr(new StaticEnv(false, env.get())); + auto newEnv = std::shared_ptr(new StaticEnv(false, env.get(), recursive ? attrs.size() : 0)); - unsigned int displ = 0; - for (auto & i : attrs) { - newEnv->vars[i.first] = i.second.displ = displ++; - } + Displacement displ = 0; + for (auto & i : attrs) + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -372,15 +383,21 @@ void ExprLambda::bindVars(const std::shared_ptr &env) if (debuggerHook) staticenv = env; - auto newEnv = std::shared_ptr(new StaticEnv(false, env.get())); + auto newEnv = std::shared_ptr( + new StaticEnv( + false, env.get(), + (hasFormals() ? formals->formals.size() : 0) + + (arg.empty() ? 0 : 1))); - unsigned int displ = 0; + Displacement displ = 0; - if (!arg.empty()) newEnv->vars[arg] = displ++; + if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); - if (matchAttrs) { + if (hasFormals()) { for (auto & i : formals->formals) - newEnv->vars[i.name] = displ++; + newEnv.vars.emplace_back(i.name, displ++); + + newEnv.sort(); for (auto & i : formals->formals) if (i.def) i.def->bindVars(newEnv); @@ -389,16 +406,28 @@ void ExprLambda::bindVars(const std::shared_ptr &env) body->bindVars(newEnv); } -void ExprLet::bindVars(const std::shared_ptr &env) +void ExprCall::bindVars(const StaticEnv & env) { if (debuggerHook) staticenv = env; - auto newEnv = std::shared_ptr(new StaticEnv(false, env.get())); + fun->bindVars(env); + for (auto e : args) + e->bindVars(env); +} - unsigned int displ = 0; +void ExprLet::bindVars(const StaticEnv & env) +{ + if (debuggerHook) + staticenv = env; + + auto newEnv = std::shared_ptr(new StaticEnv(false, env.get(), attrs->attrs.size())); + + Displacement displ = 0; for (auto & i : attrs->attrs) - newEnv->vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -415,7 +444,7 @@ void ExprWith::bindVars(const std::shared_ptr &env) level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; - unsigned int level; + Level level; prevWith = 0; for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { @@ -503,5 +532,4 @@ size_t SymbolTable::totalSize() const return n; } - } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index a78ea6215..825933fa1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -4,8 +4,6 @@ #include "symbol-table.hh" #include "error.hh" -#include - namespace nix { @@ -138,6 +136,9 @@ struct ExprPath : Expr Value * maybeThunk(EvalState & state, Env & env); }; +typedef uint32_t Level; +typedef uint32_t Displacement; + struct ExprVar : Expr { Pos pos; @@ -153,8 +154,8 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name' from the set stored in the environment that is `level' levels up from the current one.*/ - unsigned int level; - unsigned int displ; + Level level; + Displacement displ; ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; @@ -188,7 +189,7 @@ struct ExprAttrs : Expr bool inherited; Expr * e; Pos pos; - unsigned int displ; // displacement + Displacement displ; // displacement AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; @@ -236,11 +237,10 @@ struct ExprLambda : Expr Pos pos; Symbol name; Symbol arg; - bool matchAttrs; Formals * formals; Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body) + : pos(pos), arg(arg), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ @@ -250,6 +250,18 @@ struct ExprLambda : Expr }; void setName(Symbol & name); string showNamePos() const; + inline bool hasFormals() const { return formals != nullptr; } + COMMON_METHODS +}; + +struct ExprCall : Expr +{ + Expr * fun; + std::vector args; + Pos pos; + ExprCall(const Pos & pos, Expr * fun, std::vector && args) + : fun(fun), args(args), pos(pos) + { } COMMON_METHODS }; @@ -311,7 +323,6 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") @@ -345,9 +356,28 @@ struct StaticEnv { bool isWith; const StaticEnv * up; - typedef std::map Vars; + + // Note: these must be in sorted order. + typedef std::vector> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; + + StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + vars.reserve(expectedSize); + }; + + void sort() + { + std::sort(vars.begin(), vars.end(), + [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); + } + + Vars::const_iterator find(const Symbol & name) const + { + Vars::value_type key(name, 0); + auto i = std::lower_bound(vars.begin(), vars.end(), key); + if (i != vars.end() && i->first == name) return i; + return vars.end(); + } }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index d1e898677..58af0df7d 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -33,11 +33,9 @@ namespace nix { Symbol file; FileOrigin origin; std::optional error; - Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , sLetBody(symbols.create("")) { }; }; @@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs[ad.first] = ad.second; + jAttrs->attrs.emplace(ad.first, ad.second); } } else { dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); e->setName(i->symbol); } } else { @@ -283,20 +281,20 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err } %type start expr expr_function expr_if expr_op -%type expr_app expr_select expr_simple +%type expr_select expr_simple expr_app %type expr_list %type binds %type formals %type formal %type attrs attrpath %type string_parts_interpolated ind_string_parts -%type string_parts string_attr +%type path_start string_parts string_attr %type attr %token ID ATTRPATH %token STR IND_STR %token INT %token FLOAT -%token PATH HPATH SPATH +%token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW %token DOLLAR_CURLY /* == ${ */ @@ -324,13 +322,13 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); } | '{' formals '}' '@' ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); } | ID '@' '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function @@ -353,13 +351,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } @@ -367,17 +365,22 @@ expr_op | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op { $$ = new ExprConcatStrings(CUR_POS, false, new vector({$1, $3})); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } | expr_app ; expr_app - : expr_app expr_select - { $$ = new ExprApp(CUR_POS, $1, $2); } - | expr_select { $$ = $1; } + : expr_app expr_select { + if (auto e2 = dynamic_cast($1)) { + e2->args.push_back($2); + $$ = $1; + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); + } + | expr_select ; expr_select @@ -388,7 +391,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } | expr_simple { $$ = $1; } ; @@ -405,17 +408,20 @@ expr_simple | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { $$ = stripIndentation(CUR_POS, data->symbols, *$2); } - | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } + | path_start PATH_END { $$ = $1; } + | path_start string_parts_interpolated PATH_END { + $2->insert($2->begin(), $1); + $$ = new ExprConcatStrings(CUR_POS, false, $2); + } | SPATH { string path($1 + 1, strlen($1) - 2); - $$ = new ExprApp(CUR_POS, - new ExprApp(new ExprVar(data->symbols.create("__findFile")), - new ExprVar(data->symbols.create("__nixPath"))), - new ExprString(data->symbols.create(path))); + $$ = new ExprCall(CUR_POS, + new ExprVar(data->symbols.create("__findFile")), + {new ExprVar(data->symbols.create("__nixPath")), + new ExprString(data->symbols.create(path))}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); + static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), @@ -452,6 +458,20 @@ string_parts_interpolated } ; +path_start + : PATH { + Path path(absPath($1, data->basePath)); + /* add back in the trailing '/' to the first segment */ + if ($1[strlen($1)-1] == '/' && strlen($1) > 1) + path += "/"; + $$ = new ExprPath(path); + } + | HPATH { + Path path(getHome() + string($1 + 1)); + $$ = new ExprPath(path); + } + ; + ind_string_parts : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } @@ -466,7 +486,7 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } } | binds INHERIT '(' expr ')' attrs ';' @@ -475,7 +495,7 @@ binds for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } @@ -735,7 +755,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl res = { true, path }; else { logWarning({ - .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0400c8942..a9ee96bfa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,6 +21,8 @@ #include #include +#include + namespace nix { @@ -50,16 +52,13 @@ void EvalState::realiseContext(const PathSet & context) if (drvs.empty()) return; if (!evalSettings.enableImportFromDerivation) - throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", + throw Error( + "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", store->printStorePath(drvs.begin()->drvPath)); - /* For performance, prefetch all substitute info. */ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; + /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed @@ -71,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context) if (outputPaths.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store->printStorePath(drvPath), outputName); - allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + allowPath(outputPaths.at(outputName)); } } } @@ -122,7 +121,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS }); } catch (Error & e) { e.addTrace(pos, "while importing '%s'", path); - throw e; + throw; } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -158,16 +157,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS } w.attrs->sort(); - static RootValue fun; - if (!fun) { - fun = allocRootValue(state.allocValue()); + if (!state.vImportedDrvToDerivation) { + state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **fun); + , "/"), **state.vImportedDrvToDerivation); } - state.forceFunction(**fun, pos); - mkApp(v, **fun, w); + state.forceFunction(**state.vImportedDrvToDerivation, pos); + mkApp(v, **state.vImportedDrvToDerivation, w); state.forceAttrs(v, pos); } @@ -186,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - auto staticEnv = std::shared_ptr(new StaticEnv(false, state.staticBaseEnv.get())); + auto staticEnv = std::shared_ptr(new StaticEnv(false, state.staticBaseEnv.get(), vScope->attrs->size())); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv->vars[attr.name] = displ; + staticEnv.vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + printTalkative("evaluating file '%1%'", realPath); Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); @@ -412,7 +413,7 @@ static RegisterPrimOp primop_isNull({ Return `true` if *e* evaluates to `null`, and `false` otherwise. > **Warning** - > + > > This function is *deprecated*; just write `e == null` instead. )", .fun = prim_isNull, @@ -516,7 +517,11 @@ static RegisterPrimOp primop_isPath({ struct CompareValues { - bool operator () (const Value * v1, const Value * v2) const + EvalState & state; + + CompareValues(EvalState & state) : state(state) { }; + + bool operator () (Value * v1, Value * v2) const { if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; @@ -533,6 +538,17 @@ struct CompareValues return strcmp(v1->string.s, v2->string.s) < 0; case nPath: return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { + return (*this)(v1->listElems()[i], v2->listElems()[i]); + } + } default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } @@ -620,7 +636,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - set doneKeys; + auto cmp = CompareValues(state); + set doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); @@ -714,6 +731,44 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { .fun = prim_addErrorContext, }); +static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, ceil(value)); +} + +static RegisterPrimOp primop_ceil({ + .name = "__ceil", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next higher integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_ceil, +}); + +static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, floor(value)); +} + +static RegisterPrimOp primop_floor({ + .name = "__floor", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next lower integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_floor, +}); + /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -949,8 +1004,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } if (i->name == state.sContentAddressed) { - settings.requireExperimentalFeature("ca-derivations"); contentAddressed = state.forceBool(*i->value, pos); + if (contentAddressed) + settings.requireExperimentalFeature(Xp::CaDerivations); } /* The `args' attribute is special: it supplies the @@ -972,7 +1028,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, *i->value, pos, placeholder, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1134,7 +1190,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // hash per output. auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); std::visit(overloaded { - [&](Hash h) { + [&](Hash & h) { for (auto & i : outputs) { auto outPath = state.store->makeOutputPath(i, h, drvName); drv.env[i] = state.store->printStorePath(outPath); @@ -1146,11 +1202,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); } }, - [&](CaOutputHashes) { + [&](CaOutputHashes &) { // Shouldn't happen as the toplevel derivation is not CA. assert(false); }, - [&](DeferredHash _) { + [&](DeferredHash &) { for (auto & i : outputs) { drv.outputs.insert_or_assign(i, DerivationOutput { @@ -1453,15 +1509,20 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va string type = state.forceStringNoCtx(*args[0], pos); std::optional ht = parseHashType(type); if (!ht) - throw Error({ - .msg = hintfmt("unknown hash type '%1%'", type), - .errPos = pos - }); + throw Error({ + .msg = hintfmt("unknown hash type '%1%'", type), + .errPos = pos + }); - PathSet context; // discarded - Path p = state.coerceToPath(pos, *args[1], context); + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos); + } - mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context); + mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1538,7 +1599,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value { std::ostringstream out; PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); + printValueAsXML(state, true, false, *args[0], out, context, pos); mkString(v, out.str(), context); } @@ -1646,7 +1707,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu { std::ostringstream out; PathSet context; - printValueAsJSON(state, true, *args[0], out, context); + printValueAsJSON(state, true, *args[0], pos, out, context); mkString(v, out.str(), context); } @@ -1672,7 +1733,7 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va parseJSON(state, s, v); } catch (JSONParseError &e) { e.addTrace(pos, "while decoding a JSON string"); - throw e; + throw; } } @@ -1802,50 +1863,79 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); -static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) +static void addPath( + EvalState & state, + const Pos & pos, + const string & name, + Path path, + Value * filterFun, + FileIngestionMethod method, + const std::optional expectedHash, + Value & v, + const PathSet & context) { - const auto path = evalSettings.pureEval && expectedHash ? - path_ : - state.checkSourcePath(path_); - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); + try { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + state.realiseContext(context); - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); + StorePathSet refs; - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); + if (state.store->isInStore(path)) { + auto [storePath, subPath] = state.store->toStorePath(path); + // FIXME: we should scanForReferences on the path before adding it + refs = state.store->queryPathInfo(storePath)->references; + path = state.store->toRealPath(storePath) + subPath; + } - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); + path = evalSettings.pureEval && expectedHash + ? path + : state.checkSourcePath(path); - Value res; - state.callFunction(fun2, arg2, res, noPos); + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); - return state.forceBool(res, pos); - }) : defaultPathFilter; + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); - std::optional expectedStorePath; - if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); - Path dstPath; - if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); - if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) - throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); - } else - dstPath = state.store->printStorePath(*expectedStorePath); + Value arg2; + mkString(arg2, + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); - mkString(v, dstPath, {dstPath}); + Value * args []{&arg1, &arg2}; + Value res; + state.callFunction(*filterFun, 2, args, res, pos); + + return state.forceBool(res, pos); + }) : defaultPathFilter; + + std::optional expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + + Path dstPath; + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + dstPath = state.store->printStorePath(settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs)); + if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + } else + dstPath = state.store->printStorePath(*expectedStorePath); + + mkString(v, dstPath, {dstPath}); + + state.allowPath(dstPath); + + } catch (Error & e) { + e.addTrace(pos, "while adding path '%s'", path); + throw; + } } @@ -1853,11 +1943,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args { PathSet context; Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = pos - }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) @@ -1868,13 +1953,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ .name = "__filterSource", .args = {"e1", "e2"}, .doc = R"( + > **Warning** + > + > `filterSource` should not be used to filter store paths. Since + > `filterSource` uses the name of the input directory while naming + > the output directory, doing so will produce a directory name in + > the form of `--`, where `-` is + > the name of the input directory. Since `` depends on the + > unfiltered directory, the name of the output directory will + > indirectly depend on files that are filtered out by the + > function. This will trigger a rebuild even when a filtered out + > file is changed. Use `builtins.path` instead, which allows + > specifying the name of the output directory. + This function allows you to copy sources into the Nix store while filtering certain files. For instance, suppose that you want to use the directory `source-dir` as an input to a Nix expression, e.g. @@ -1921,18 +2019,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; + PathSet context; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); - if (n == "path") { - PathSet context; + if (n == "path") path = state.coerceToPath(*attr.pos, *attr.value, context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = *attr.pos - }); - } else if (attr.name == state.sName) + else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { state.forceValue(*attr.value, pos); @@ -1955,7 +2048,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, method, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2069,7 +2162,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) pos ); // !!! add to stack trace? - if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; + if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++; state.forceValue(*i->value, pos); v = *i->value; } @@ -2320,7 +2413,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - if (!args[0]->lambda.fun->matchAttrs) { + if (!args[0]->lambda.fun->hasFormals()) { state.mkAttrs(v, 0); return; } @@ -2329,7 +2422,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) Value * value = state.allocValue(); - v.attrs->push_back(Attr(i.name, value, &i.pos)); + v.attrs->push_back(Attr(i.name, value, ptr(&i.pos))); mkBool(*value, i.def); } v.attrs->sort(); @@ -2475,7 +2568,7 @@ static RegisterPrimOp primop_tail({ the argument isn’t a list or is an empty list. > **Warning** - > + > > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. @@ -2617,10 +2710,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value * vCur = args[1]; for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); + Value * vs []{vCur, args[2]->listElems()[n]}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + state.callFunction(*args[0], 2, vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -2634,9 +2726,9 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. )", .fun = prim_foldlStrict, @@ -2741,17 +2833,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[1]->listElems()[n]; } - auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + return CompareValues(state)(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); + Value * vs[] = {a, b}; + Value vBool; + state.callFunction(*args[0], 2, vs, vBool, pos); + return state.forceBool(vBool, pos); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -2857,7 +2948,7 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); } catch (TypeError &e) { e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap")); - throw e; + throw; } len += lists[n].listSize(); } @@ -3028,7 +3119,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp; + CompareValues comp{state}; mkBool(v, comp(args[0], args[1])); } @@ -3069,7 +3160,7 @@ static RegisterPrimOp primop_toString({ - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`. - - A set containing `{ __toString = self: ...; }`. + - A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`. - An integer. @@ -3154,7 +3245,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(*ht, s).to_string(Base16, false), context); + mkString(v, hashString(*ht, s).to_string(Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3561,15 +3652,13 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun, - std::optional requiredFeature) +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) { if (!primOps) primOps = new PrimOps; primOps->push_back({ .name = name, .args = {}, .arity = arity, - .requiredFeature = std::move(requiredFeature), .fun = fun }); } @@ -3605,9 +3694,7 @@ void EvalState::createBaseEnv() if (!evalSettings.pureEval) { mkInt(v, time(0)); addConstant("__currentTime", v); - } - if (!evalSettings.pureEval) { mkString(v, settings.thisSystem.get()); addConstant("__currentSystem", v); } @@ -3622,7 +3709,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 5); + mkInt(v, 6); addConstant("__langVersion", v); // Miscellaneous @@ -3645,26 +3732,31 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature)) - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = symbols.create(primOp.name), - .args = std::move(primOp.args), - .doc = primOp.doc, - }); + addPrimOp({ + .fun = primOp.fun, + .arity = std::max(primOp.args.size(), primOp.arity), + .name = symbols.create(primOp.name), + .args = std::move(primOp.args), + .doc = primOp.doc, + }); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ sDerivationNix = symbols.create("//builtin/derivation.nix"); - eval(parse( - #include "primops/derivation.nix.gen.hh" - , foFile, sDerivationNix, "/", staticBaseEnv), v); - addConstant("derivation", v); + auto vDerivation = allocValue(); + addConstant("derivation", vDerivation); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); + + staticBaseEnv.sort(); + + /* Note: we have to initialize the 'derivation' constant *after* + building baseEnv/staticBaseEnv because it uses 'builtins'. */ + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 9d42d6539..5b16e075f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -15,7 +15,6 @@ struct RegisterPrimOp std::vector args; size_t arity = 0; const char * doc; - std::optional requiredFeature; PrimOpFun fun; }; @@ -28,8 +27,7 @@ struct RegisterPrimOp RegisterPrimOp( std::string name, size_t arity, - PrimOpFun fun, - std::optional requiredFeature = {}); + PrimOpFun fun); RegisterPrimOp(Info && info); }; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 4830ebec3..c23480853 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -15,7 +15,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar std::string name = "source"; PathSet context; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { @@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar fetchers::Attrs attrs; attrs.insert_or_assign("type", "hg"); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + attrs.insert_or_assign("name", name); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); @@ -83,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); } static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b8b99d4fa..079513873 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -15,7 +16,8 @@ void emitTreeAttrs( const fetchers::Tree & tree, const fetchers::Input & input, Value & v, - bool emptyRevFallback) + bool emptyRevFallback, + bool forceDirty) { assert(input.isImmutable()); @@ -32,24 +34,28 @@ void emitTreeAttrs( mkString(*state.allocAttr(v, state.symbols.create("narHash")), narHash->to_string(SRI, true)); - if (auto rev = input.getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); - } else if (emptyRevFallback) { - // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); - mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); - } - if (input.getType() == "git") mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); - if (auto revCount = input.getRevCount()) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - else if (emptyRevFallback) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + if (!forceDirty) { + + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); + } + + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + + } if (auto lastModified = input.getLastModified()) { mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); @@ -60,47 +66,74 @@ void emitTreeAttrs( v.attrs->sort(); } -std::string fixURI(std::string uri, EvalState &state) +std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : "file://" + uri; + return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; } -void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +std::string fixURIForGit(std::string uri, EvalState & state) { - string n(name); - attrs.emplace(name, n == "url" ? fixURI(v, state) : v); + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ + static std::regex scp_uri("([^/]*)@(.*):(.*)"); + if (uri[0] != '/' && std::regex_match(uri, scp_uri)) + return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); + else + return fixURI(uri, state); } +struct FetchTreeParams { + bool emptyRevFallback = false; + bool allowNameArgument = false; +}; + static void fetchTree( - EvalState &state, - const Pos &pos, - Value **args, - Value &v, - const std::optional type, - bool emptyRevFallback = false + EvalState & state, + const Pos & pos, + Value * * args, + Value & v, + std::optional type, + const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; PathSet context; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; + if (auto aType = args[0]->attrs->get(state.sType)) { + if (type) + throw Error({ + .msg = hintfmt("unexpected attribute 'type'"), + .errPos = pos + }); + type = state.forceStringNoCtx(*aType->value, *aType->pos); + } else if (!type) + throw Error({ + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .errPos = pos + }); + + attrs.emplace("type", type.value()); + for (auto & attr : *args[0]->attrs) { - state.forceValue(*attr.value); - if (attr.value->type() == nPath || attr.value->type() == nString) - addURI( - state, - attrs, - attr.name, - state.coerceToString(*attr.pos, *attr.value, context, false, false) - ); - else if (attr.value->type() == nString) - addURI(state, attrs, attr.name, attr.value->string.s); + if (attr.name == state.sType) continue; + state.forceValue(*attr.value, *attr.pos); + if (attr.value->type() == nPath || attr.value->type() == nString) { + auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false); + attrs.emplace(attr.name, + attr.name == "url" + ? type == "git" + ? fixURIForGit(s, state) + : fixURI(s, state) + : s); + } else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) @@ -110,26 +143,24 @@ static void fetchTree( attr.name, showType(*attr.value)); } - if (type) - attrs.emplace("type", type.value()); - - if (!attrs.count("type")) - throw Error({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = pos - }); + if (!params.allowNameArgument) + if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) + throw Error({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = pos + }); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + auto url = state.coerceToString(pos, *args[0], context, false, false); if (type == "git") { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", url); + attrs.emplace("url", fixURIForGit(url, state)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(fixURI(url, state)); } } @@ -141,16 +172,15 @@ static void fetchTree( auto [tree, input2] = input.fetch(state.store); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); - emitTreeAttrs(state, tree, input2, v, emptyRevFallback); + emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) { - settings.requireExperimentalFeature("flakes"); - fetchTree(state, pos, args, v, std::nullopt); + settings.requireExperimentalFeature(Xp::Flakes); + fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } // FIXME: document @@ -162,7 +192,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, std::optional url; std::optional expectedHash; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { @@ -206,20 +236,18 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; - auto path = state.store->toRealPath(storePath); - if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, path); + : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } - if (state.allowedPaths) - state.allowedPaths->insert(path); + state.allowPath(storePath); + auto path = state.store->printStorePath(storePath); mkString(v, path, PathSet({path})); } @@ -262,13 +290,13 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - The fetched tarball is cached for a certain amount of time (1 hour - by default) in `~/.cache/nix/tarballs/`. You can change the cache - timeout either on the command line with `--option tarball-ttl number - of seconds` or in the Nix configuration file with this option: ` - number of seconds to cache `. + The fetched tarball is cached for a certain amount of time (1 + hour by default) in `~/.cache/nix/tarballs/`. You can change the + cache timeout either on the command line with `--tarball-ttl` + *number-of-seconds* or in the Nix configuration file by adding + the line `tarball-ttl = ` *number-of-seconds*. - Note that when obtaining the hash with ` nix-prefetch-url ` the + Note that when obtaining the hash with `nix-prefetch-url` the option `--unpack` is required. This function can also verify the contents against a hash. In that @@ -292,7 +320,7 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) { - fetchTree(state, pos, args, v, "git", true); + fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } static RegisterPrimOp primop_fetchGit({ @@ -368,7 +396,7 @@ static RegisterPrimOp primop_fetchGit({ ``` > **Note** - > + > > It is nice to always specify the branch which a revision > belongs to. Without the branch being specified, the fetcher > might fail if the default branch changes. Additionally, it can @@ -405,12 +433,12 @@ static RegisterPrimOp primop_fetchGit({ ``` > **Note** - > + > > Nix will refetch the branch in accordance with > the option `tarball-ttl`. > **Note** - > + > > This behavior is disabled in *Pure evaluation mode*. )", .fun = prim_fetchGit, diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index bfea24d40..4d642c720 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,11 +10,11 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { out.write(*maybeString); break; @@ -54,10 +54,10 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); + printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); } } else - printValueAsJSON(state, strict, *i->value, out, context); + printValueAsJSON(state, strict, *i->value, *i->pos, out, context); break; } @@ -65,7 +65,7 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + printValueAsJSON(state, strict, *v.listElems()[n], pos, placeholder, context); } break; } @@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError("cannot convert %1% to JSON", showType(v)); - case nFunction: - throw TypeError("cannot convert %1% to JSON", showType(v)); + auto e = TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = v.determinePos(pos) + }); + throw e.addTrace(pos, hintfmt("message for the trace")); } } void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, const Pos & pos, std::ostream & str, PathSet & context) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); + printValueAsJSON(state, strict, v, pos, out, context); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 67fed6487..c2f797b29 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); + Value & v, const Pos & pos, std::ostream & str, PathSet & context); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 7464455d8..54268ece0 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -42,21 +43,22 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLAttrs xmlAttrs; xmlAttrs["name"] = i; - if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); + if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos); XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); + *a.value, doc, context, drvsSeen, *a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -121,7 +123,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen, pos); break; } @@ -135,7 +137,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->matchAttrs) { + if (v.lambda.fun->hasFormals()) { XMLAttrs attrs; if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; @@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: @@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, PathSet & context, const Pos & pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..cc778a2cb 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - + Value & v, std::ostream & out, PathSet & context, const Pos & pos); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a1f131f9e..3bb97b3c2 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -94,7 +94,8 @@ class ExternalValueBase /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const; virtual ~ExternalValueBase() { diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index a2d53a7bf..e41037633 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -6,6 +6,8 @@ #include +#include + namespace nix::fetchers { typedef std::variant> Attr; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 916e0a8e8..e158d914b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -200,12 +200,17 @@ void Input::markChangedFile( return scheme->markChangedFile(*this, file, commitMsg); } +std::string Input::getName() const +{ + return maybeGetStrAttr(attrs, "name").value_or("source"); +} + StorePath Input::computeStorePath(Store & store) const { auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); } std::string Input::getType() const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a72cfafa4..c43b047a7 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -38,6 +38,9 @@ struct Input bool immutable = false; bool direct = true; + /* path of the parent of this input, used for relative path resolution */ + std::optional parent; + public: static Input fromURL(const std::string & url); @@ -81,6 +84,8 @@ public: std::string_view file, std::optional commitMsg) const; + std::string getName() const; + StorePath computeStorePath(Store & store) const; // Convenience functions for common attributes. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d8e0dbe0a..544d2ffbf 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -4,6 +4,7 @@ #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" +#include "pathlocks.hh" #include #include @@ -12,6 +13,12 @@ using namespace std::string_literals; namespace nix::fetchers { +// Explicit initial branch of our bare repo to suppress warnings from new version of git. +// The value itself does not matter, since we always fetch a specific revision or branch. +// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with +// old version of git, which will ignore unrecognized `-c` options. +const std::string gitInitialBranch = "__nix_dummy_branch"; + static std::string readHead(const Path & path) { return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); @@ -44,7 +51,7 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow") + else if (name == "shallow" || name == "submodules") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -60,7 +67,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -167,10 +174,10 @@ struct GitInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { - auto name = "source"; - Input input(_input); + std::string name = input.getName(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); @@ -270,7 +277,7 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? @@ -317,9 +324,11 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; + createDirs(dirOf(cacheDir)); + PathLocks cacheDirLock({cacheDir + ".lock"}); + if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", repoDir }); + runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } Path localRefFile = @@ -386,6 +395,8 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + + // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -406,17 +417,14 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; - RunOptions checkCommitOpts( - "git", - { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() } - ); - checkCommitOpts.searchPath = true; - checkCommitOpts.mergeStderrToStdout = true; - - auto result = runProgram(checkCommitOpts); + auto result = runProgram(RunOptions { + .program = "git", + .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }, + .mergeStderrToStdout = true + }); if (WEXITSTATUS(result.first) == 128 - && result.second.find("bad file") != std::string::npos - ) { + && result.second.find("bad file") != std::string::npos) + { throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " @@ -432,7 +440,7 @@ struct GitInputScheme : InputScheme Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); - runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); // TODO: repoDir might lack the ref (it only checks if rev // exists, see FIXME above) so use a big hammer and fetch // everything to ensure we get the rev. @@ -448,9 +456,11 @@ struct GitInputScheme : InputScheme // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); + runProgram2({ + .program = "git", + .args = { "-C", repoDir, "archive", input.getRev()->gitRev() }, + .standardOut = &sink + }); }); unpackTarfile(*source, tmpDir); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8352ef02d..1c539b80e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); @@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + Input::fromURL(fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) - .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } }; @@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme if ("PAT" == token.substr(0, fldsplit)) return std::make_pair("Private-token", token.substr(fldsplit+1)); warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); - return std::nullopt; + return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } Hash getRevFromRef(nix::ref store, const Input & input) const override @@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + Input::fromURL(fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) - .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } }; diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index cfd705e22..2e8869d83 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore +libfetchers_LDFLAGS += -pthread + libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 0eb401e10..d52d4641b 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -11,34 +11,32 @@ using namespace std::string_literals; namespace nix::fetchers { -namespace { +static RunOptions hgOptions(const Strings & args) +{ + auto env = getEnv(); + // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc. + env["HGPLAIN"] = ""; -RunOptions hgOptions(const Strings & args) { - RunOptions opts("hg", args); - opts.searchPath = true; - - auto env = getEnv(); - // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc. - env["HGPLAIN"] = ""; - opts.environment = env; - - return opts; + return { + .program = "hg", + .searchPath = true, + .args = args, + .environment = env + }; } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -string runHg(const Strings & args, const std::optional & input = {}) +static string runHg(const Strings & args, const std::optional & input = {}) { - RunOptions opts = hgOptions(args); - opts.input = input; + RunOptions opts = hgOptions(args); + opts.input = input; - auto res = runProgram(opts); + auto res = runProgram(std::move(opts)); - if (!statusOk(res.first)) - throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); - - return res.second; -} + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + return res.second; } struct MercurialInputScheme : InputScheme @@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "hg") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") throw Error("unsupported Mercurial input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { - auto name = "source"; - Input input(_input); + auto name = input.getName(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug @@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return { Tree(store->toRealPath(storePath), std::move(storePath)), @@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme have to pull again. */ if (!(input.getRev() && pathExists(cacheDir) - && runProgram( - hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) - .killStderr(true)).second == "1")) + && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d1003de57..fb5702c4c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme std::pair fetch(ref store, const Input & input) override { + std::string absPath; auto path = getStrAttr(input.attrs, "path"); - // FIXME: check whether access to 'path' is allowed. + if (path[0] != '/') { + if (!input.parent) + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); - auto storePath = store->maybeParseStorePath(path); + auto parent = canonPath(*input.parent); + + // the path isn't relative, prefix it + absPath = nix::absPath(path, parent); + + // for security, ensure that if the parent is a store path, it's inside it + if (store->isInStore(parent)) { + auto storePath = store->printStorePath(store->toStorePath(parent).first); + if (!isInDir(absPath, storePath)) + throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); + } + } else + absPath = path; + + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); + + // FIXME: check whether access to 'path' is allowed. + auto storePath = store->maybeParseStorePath(absPath); if (storePath) store->addTempRoot(*storePath); if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) // FIXME: try to substitute storePath. - storePath = store->addToStore("source", path); + storePath = store->addToStore("source", absPath); return { Tree(store->toRealPath(*storePath), std::move(*storePath)), diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 74376adc0..f35359d4b 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -124,6 +124,13 @@ std::shared_ptr getUserRegistry() return userRegistry; } +std::shared_ptr getCustomRegistry(const Path & p) +{ + static auto customRegistry = + Registry::read(p, Registry::Custom); + return customRegistry; +} + static std::shared_ptr flagRegistry = std::make_shared(Registry::Flag); diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 1077af020..260a2c460 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -14,6 +14,7 @@ struct Registry User = 1, System = 2, Global = 3, + Custom = 4, }; RegistryType type; @@ -48,6 +49,8 @@ typedef std::vector> Registries; std::shared_ptr getUserRegistry(); +std::shared_ptr getCustomRegistry(const Path & p); + Path getUserRegistryPath(); Registries getRegistries(ref store); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b8d7d2c70..031ccc5f7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2")) + && !hasSuffix(url.path, ".tar.bz2") + && !hasSuffix(url.path, ".tar.zst")) return {}; Input input; @@ -195,7 +196,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") throw Error("unsupported tarball input attribute '%s'", name); Input input; @@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme std::pair fetch(ref store, const Input & input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; return {std::move(tree), input}; } }; diff --git a/src/libmain/local.mk b/src/libmain/local.mk index a8eed6c65..99da95e27 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -8,10 +8,10 @@ libmain_SOURCES := $(wildcard $(d)/*.cc) libmain_CXXFLAGS += -I src/libutil -I src/libstore -libmain_LDFLAGS = $(OPENSSL_LIBS) +libmain_LDFLAGS += $(OPENSSL_LIBS) libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 15354549a..63955eed1 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -103,17 +103,19 @@ public: ~ProgressBar() { stop(); - updateThread.join(); } void stop() override { - auto state(state_.lock()); - if (!state->active) return; - state->active = false; - writeToStderr("\r\e[K"); - updateCV.notify_one(); - quitCV.notify_one(); + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + writeToStderr("\r\e[K"); + updateCV.notify_one(); + quitCV.notify_one(); + } + updateThread.join(); } bool isVerbose() override { @@ -484,7 +486,7 @@ Logger * makeProgressBar(bool printBuildLogs) { return new ProgressBar( printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + shouldANSI() ); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 86930c2e3..85f9f0d58 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,6 +15,9 @@ #include #include #include +#include +#include +#include #include @@ -110,6 +113,31 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) } #endif +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. + We need the resolution to be done locally, as nscd socket will not be accessible in the + sandbox. */ + char * previous_env = getenv("LOCALDOMAIN"); + setenv("LOCALDOMAIN", "invalid", 1); + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { + if (res) freeaddrinfo(res); + } + if (previous_env) { + setenv("LOCALDOMAIN", previous_env, 1); + } else { + unsetenv("LOCALDOMAIN"); + } + }); +} static void sigHandler(int signo) { } @@ -176,6 +204,8 @@ void initNix() if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) unsetenv("TMPDIR"); #endif + + preloadNSS(); } @@ -238,7 +268,7 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-gc-warning", .description = "Disable warnings about not using `--add-root`.", - .handler = {&gcWarning, true}, + .handler = {&gcWarning, false}, }); addFlag({ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 09e1c254b..13c086a46 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -52,9 +52,9 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1" ? "true" : "false"); + wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { - priority.setDefault(fmt("%d", std::stoi(value))); + priority.setDefault(std::stoi(value)); } } } @@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - std::string hashPart(narInfo->path.hashPart()); - { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr(narInfo) }); + state_->pathInfoCache.upsert( + std::string(narInfo->path.to_string()), + PathInfoCacheValue { .value = std::shared_ptr(narInfo) }); } if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); + diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } AutoCloseFD openFile(const Path & path) @@ -130,17 +130,6 @@ AutoCloseFD openFile(const Path & path) return fd; } -struct FileSource : FdSource -{ - AutoCloseFD fd2; - - FileSource(const Path & path) - : fd2(openFile(path)) - { - fd = fd2.get(); - } -}; - ref BinaryCacheStore::addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo) @@ -160,7 +149,7 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); + auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -319,16 +308,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + makeFixedOutputPath(method, nar.first, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; return info; })->path; } @@ -396,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -415,10 +405,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + makeFixedOutputPath(method, h, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = h, @@ -448,20 +439,34 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } -std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) +void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, + Callback> callback) noexcept { auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; - auto rawOutputInfo = getFile(outputInfoFilePath); - if (rawOutputInfo) { - return {Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; - } else { - return std::nullopt; - } + auto callbackPtr = std::make_shared(std::move(callback)); + + Callback> newCallback = { + [=](std::future> fut) { + try { + auto data = fut.get(); + if (!data) return (*callbackPtr)(nullptr); + + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*data), outputInfoFilePath); + return (*callbackPtr)(std::make_shared(realisation)); + } catch (...) { + callbackPtr->rethrow(); + } + } + }; + + getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + if (diskCache) + diskCache->upsertRealisation(getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c2163166c..9815af591 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression, available for xz only currently"}; + "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + "specify 'preset level' of compression to be used with NARs: " + "meaning and accepted range of values depends on compression method selected, " + "other than -1 which we reserve to indicate Nix defaults should be used"}; }; class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store @@ -34,7 +38,7 @@ private: protected: // The prefix under which realisation infos will be stored - const std::string realisationsPrefix = "/realisations"; + const std::string realisationsPrefix = "realisations"; BinaryCacheStore(const Params & params); @@ -93,18 +97,19 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override; void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 9100d3333..b924d23b2 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -143,7 +143,6 @@ void DerivationGoal::work() (this->*state)(); } - void DerivationGoal::addWantedOutputs(const StringSet & outputs) { /* If we already want all outputs, there is nothing to do. */ @@ -166,7 +165,7 @@ void DerivationGoal::getDerivation() /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { loadDerivation(); return; } @@ -189,12 +188,12 @@ void DerivationGoal::loadDerivation() /* `drvPath' should already be a root, but let's be on the safe side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); + worker.evalStore.addTempRoot(drvPath); - assert(worker.store.isValidPath(drvPath)); + assert(worker.evalStore.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::make_unique(worker.store.derivationFromPath(drvPath)); + drv = std::make_unique(worker.evalStore.derivationFromPath(drvPath)); haveDerivation(); } @@ -205,7 +204,7 @@ void DerivationGoal::haveDerivation() trace("have derivation"); if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); retrySubstitution = false; @@ -213,8 +212,8 @@ void DerivationGoal::haveDerivation() if (i.second.second) worker.store.addTempRoot(*i.second.second); - auto outputHashes = staticOutputHashes(worker.store, *drv); - for (auto &[outputName, outputHash] : outputHashes) + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + for (auto & [outputName, outputHash] : outputHashes) initialOutputs.insert({ outputName, InitialOutput{ @@ -338,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + /* Copy the input sources from the eval store to the build + store. */ + if (&worker.evalStore != &worker.store) { + RealisedPath::Set inputSrcs; + for (auto & i : drv->inputSrcs) + inputSrcs.insert(i); + copyClosure(worker.evalStore, worker.store, inputSrcs); + } + for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) @@ -445,7 +453,7 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if (settings.isExperimentalFeatureEnabled("ca-derivations") && + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) || fullDrv.type() == DerivationType::DeferredInputAddressed)) { /* We are be able to resolve this derivation based on the @@ -479,8 +487,8 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + assert(worker.evalStore.isValidPath(drvPath)); + auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); for (auto & j : wantedDepOutputs) { if (outputs.count(j) > 0) { auto optRealizedInput = outputs.at(j); @@ -545,7 +553,7 @@ void DerivationGoal::tryToBuild() PathSet lockFiles; /* FIXME: Should lock something like the drv itself so we don't build same CA drv concurrently */ - if (dynamic_cast(&worker.store)) + if (dynamic_cast(&worker.store)) { /* If we aren't a local store, we might need to use the local store as a build remote, but that would cause a deadlock. */ /* FIXME: Make it so we can use ourselves as a build remote even if we @@ -553,9 +561,15 @@ void DerivationGoal::tryToBuild() /* FIXME: find some way to lock for scheduling for the other stores so a forking daemon with --store still won't farm out redundant builds. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) + for (auto & i : drv->outputsAndOptPaths(worker.store)) { if (i.second.second) lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + else + lockFiles.insert( + worker.store.Store::toRealPath(drvPath) + "." + i.first + ); + } + } if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -602,7 +616,9 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + bool buildLocally = + (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store)) + && settings.maxBuildJobs.get() != 0; if (!buildLocally) { switch (tryBuildHook()) { @@ -739,6 +755,64 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() { } +void runPostBuildHook( + Store & store, + Logger & logger, + const StorePath & drvPath, + StorePathSet outputPaths +) +{ + auto hook = settings.postBuildHook; + if (hook == "") + return; + + Activity act(logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{store.printStorePath(drvPath)}); + PushActivity pact(act.id); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); + hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue()); + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (std::string_view data) override { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + runProgram2({ + .program = settings.postBuildHook, + .environment = hookEnvironment, + .standardOut = &sink, + .mergeStderrToStdout = true, + }); +} void DerivationGoal::buildDone() { @@ -804,57 +878,15 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (std::string_view data) override { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } + StorePathSet outputPaths; + for (auto & [_, path] : finalOutputs) + outputPaths.insert(path); + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); if (buildMode == bmCheck) { cleanupPostOutputsRegisteredModeCheck(); @@ -910,6 +942,8 @@ void DerivationGoal::resolvedFinished() { auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + StorePathSet outputPaths; + // `wantedOutputs` might be empty, which means “all the outputs” auto realWantedOutputs = wantedOutputs; if (realWantedOutputs.empty()) @@ -927,8 +961,10 @@ void DerivationGoal::resolvedFinished() { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.signatures.clear(); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); + outputPaths.insert(realisation->outPath); } else { // If we don't have a realisation, then it must mean that something // failed when building the resolved drv @@ -936,6 +972,13 @@ void DerivationGoal::resolvedFinished() { } } + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + // This is potentially a bit fishy in terms of error reporting. Not sure // how to do it in a cleaner way amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); @@ -968,7 +1011,7 @@ HookReply DerivationGoal::tryBuildHook() return readLine(worker.hook->fromHook.readSide.get()); } catch (Error & e) { e.addTrace({}, "while reading the response from the build hook"); - throw e; + throw; } }(); if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) @@ -1014,7 +1057,7 @@ HookReply DerivationGoal::tryBuildHook() machineName = readLine(hook->fromHook.readSide.get()); } catch (Error & e) { e.addTrace({}, "while reading the machine name from the build hook"); - throw e; + throw; } /* Tell the hook all the inputs that have to be copied to the @@ -1048,42 +1091,6 @@ HookReply DerivationGoal::tryBuildHook() } -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -1268,7 +1275,7 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index a5ac4c49d..b9602e696 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,6 +1,8 @@ #include "drv-output-substitution-goal.hh" +#include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "callback.hh" namespace nix { @@ -17,6 +19,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker void DrvOutputSubstitutionGoal::init() { trace("init"); + + /* If the derivation already exists, we’re done */ + if (worker.store.queryRealisation(id)) { + amDone(ecSuccess); + return; + } + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); tryNext(); } @@ -43,14 +52,62 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - auto sub = subs.front(); + sub = subs.front(); subs.pop_front(); // FIXME: Make async - outputInfo = sub->queryRealisation(id); + // outputInfo = sub->queryRealisation(id); + outPipe.create(); + promise = decltype(promise)(); + + sub->queryRealisation( + id, { [&](std::future> res) { + try { + Finally updateStats([this]() { outPipe.writeSide.close(); }); + promise.set_value(res.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &DrvOutputSubstitutionGoal::realisationFetched; +} + +void DrvOutputSubstitutionGoal::realisationFetched() +{ + worker.childTerminated(this); + + try { + outputInfo = promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + if (!outputInfo) { - tryNext(); - return; + return tryNext(); + } + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + tryNext(); + return; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + } } addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); @@ -92,4 +149,10 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } +void DrvOutputSubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 63ab53d89..67ae2624a 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -3,6 +3,8 @@ #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include +#include namespace nix { @@ -20,11 +22,18 @@ private: // The realisation corresponding to the given output id. // Will be filled once we can get it. - std::optional outputInfo; + std::shared_ptr outputInfo; /* The remaining substituters. */ std::list> subs; + /* The current substituter. */ + std::shared_ptr sub; + + Pipe outPipe; + std::thread thr; + std::promise> promise; + /* Whether a substituter failed. */ bool substituterFailed = false; @@ -36,6 +45,7 @@ public: void init(); void tryNext(); + void realisationFetched(); void outPathValid(); void finished(); @@ -44,7 +54,7 @@ public: string key() override; void work() override; - + void handleEOF(int fd) override; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 732d4785d..9b4cfd835 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,4 +1,3 @@ -#include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" @@ -6,17 +5,17 @@ namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) { - Worker worker(*this); + Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (auto & br : reqs) { + for (const auto & br : reqs) { std::visit(overloaded { - [&](DerivedPath::Built bfd) { + [&](const DerivedPath::Built & bfd) { goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); }, - [&](DerivedPath::Opaque bo) { + [&](const DerivedPath::Opaque & bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, }, br.raw()); @@ -51,7 +50,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this); + Worker worker(*this, *this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; @@ -74,7 +73,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat outputId, Realisation{ outputId, *staticOutput.second} ); - if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) { auto realisation = this->queryRealisation(outputId); if (realisation) result.builtOutputs.insert_or_assign( @@ -93,7 +92,7 @@ void Store::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; @@ -111,7 +110,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 9de40bdf2..7c985128b 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -13,11 +13,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { void addToWeakGoals(WeakGoals & goals, GoalPtr p) { - // FIXME: necessary? - // FIXME: O(n) - for (auto & i : goals) - if (i.lock() == p) return; - goals.push_back(p); + if (goals.find(p) != goals.end()) + return; + goals.insert(p); } @@ -46,10 +44,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) /* If we failed and keepGoing is not set, we remove all remaining waitees. */ for (auto & goal : waitees) { - WeakGoals waiters2; - for (auto & j : goal->waiters) - if (j.lock() != shared_from_this()) waiters2.push_back(j); - goal->waiters = waiters2; + goal->waiters.extract(shared_from_this()); } waitees.clear(); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index e6bf628cb..192e416d2 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -19,7 +19,7 @@ struct CompareGoalPtrs { /* Set of goals. */ typedef set Goals; -typedef list WeakGoals; +typedef set> WeakGoals; /* A map of paths to goals (and the other way around). */ typedef std::map WeakGoalMap; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7c1402918..c9a4a31e7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -17,16 +17,14 @@ #include #include -#include -#include #include -#include #include #include #include #include #include #include +#include #if HAVE_STATVFS #include @@ -34,7 +32,6 @@ /* Includes required for chroot support. */ #if __linux__ -#include #include #include #include @@ -70,12 +67,14 @@ void handleDiffHook( auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - auto diffRes = runProgram(diffHookOptions); + auto diffRes = runProgram(RunOptions { + .program = diffHook, + .searchPath = true, + .args = {tryA, tryB, drvPath, tmpDir}, + .uid = uid, + .gid = gid, + .chdir = "/" + }); if (!statusOk(diffRes.first)) throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", @@ -153,6 +152,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { + state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -291,7 +291,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -342,24 +342,7 @@ int childEntry(void * arg) return 1; } - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - +#if __linux__ static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -375,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to) copyPath(from, to); } } +#endif void LocalDerivationGoal::startBuilder() @@ -388,9 +372,6 @@ void LocalDerivationGoal::startBuilder() settings.thisSystem, concatStringsSep(", ", worker.store.systemFeatures)); - if (drv->isBuiltin()) - preloadNSS(); - #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif @@ -416,7 +397,7 @@ void LocalDerivationGoal::startBuilder() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir) { + if (localStore.storeDir != localStore.realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -517,7 +498,7 @@ void LocalDerivationGoal::startBuilder() /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + worker.store.exportReferences({storePath}, inputPaths), false, false)); } } @@ -581,7 +562,9 @@ void LocalDerivationGoal::startBuilder() throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", worker.store.printStorePath(drvPath), i); - dirsInChroot[i] = i; + /* Allow files in __impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + dirsInChroot[i] = {i, true}; } #if __linux__ @@ -729,6 +712,7 @@ void LocalDerivationGoal::startBuilder() if (!builderOut.readSide) throw SysError("opening pseudoterminal master"); + // FIXME: not thread-safe, use ptsname_r std::string slaveName(ptsname(builderOut.readSide.get())); if (buildUser) { @@ -772,7 +756,6 @@ void LocalDerivationGoal::startBuilder() result.startTime = time(0); /* Fork a child to build the package. */ - ProcessOptions options; #if __linux__ if (useChroot) { @@ -815,8 +798,6 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - options.allowVfork = false; - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; static bool userNamespacesEnabled = pathExists(maxUserNamespaces) @@ -874,7 +855,7 @@ void LocalDerivationGoal::startBuilder() writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); - }, options); + }); int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { @@ -937,11 +918,12 @@ void LocalDerivationGoal::startBuilder() } else #endif { +#if __linux__ fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); +#endif pid = startProcess([&]() { runChild(); - }, options); + }); } /* parent */ @@ -956,9 +938,12 @@ void LocalDerivationGoal::startBuilder() try { return readLine(builderOut.readSide.get()); } catch (Error & e) { - e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + auto status = pid.wait(); + e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", + worker.store.printStorePath(drvPath), + statusToString(status), concatStringsSep("|", msgs)); - throw e; + throw; } }(); if (string(msg, 0, 1) == "\2") break; @@ -1081,123 +1066,38 @@ void LocalDerivationGoal::initEnv() } -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - void LocalDerivationGoal::writeStructuredAttrs() { - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { + auto json = structAttrsJson.value(); + nlohmann::json rewritten; + for (auto & [i, v] : json["outputs"].get()) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + rewritten[i] = rewriteStrings(v, inputRewrites); + } - auto json = *structuredAttrs; + json["outputs"] = rewritten; - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + auto jsonSh = writeStructuredAttrsShell(json); + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); } static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { - [&](DerivedPath::Opaque bo) { + [&](const DerivedPath::Opaque & bo) { return bo.path; }, - [&](DerivedPath::Built bfd) { + [&](const DerivedPath::Built & bfd) { return bfd.drvPath; }, }, req.raw()); @@ -1282,7 +1182,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1301,9 +1202,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo } StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); return path; } @@ -1327,16 +1229,24 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // corresponds to an allowed derivation { throw Error("registerDrvOutput"); } - std::optional queryRealisation(const DrvOutput & id) override + void queryRealisationUncached(const DrvOutput & id, + Callback> callback) noexcept override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation - { throw Error("queryRealisation"); } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override { + if (!goal.isAllowed(id)) + callback(nullptr); + next->queryRealisation(id, std::move(callback)); + } + + void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override + { + assert(!evalStore); + if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; + std::set newRealisations; for (auto & req : paths) { if (!goal.isAllowed(req)) @@ -1349,16 +1259,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo auto p = std::get_if(&path); if (!p) continue; auto & bfd = *p; + auto drv = readDerivation(bfd.drvPath); + auto drvHashes = staticOutputHashes(*this, drv); auto outputs = next->queryDerivationOutputMap(bfd.drvPath); for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) + if (wantOutput(outputName, bfd.outputs)) { newPaths.insert(outputPath); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto thisRealisation = next->queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName} + ); + assert(thisRealisation); + newRealisations.insert(*thisRealisation); + } + } } StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -1404,7 +1326,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature("recursive-nix"); + settings.requireExperimentalFeature(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -1437,7 +1359,7 @@ void LocalDerivationGoal::startDaemon() AutoCloseFD remote = accept(daemonSocket.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { - if (errno == EINTR) continue; + if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINVAL) break; throw SysError("accepting connection"); } @@ -1734,7 +1656,7 @@ void LocalDerivationGoal::runChild() /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations within a pure derivation. */ - for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); } @@ -1916,7 +1838,7 @@ void LocalDerivationGoal::runChild() /* Fill in the arguments. */ Strings args; - const char *builder = "invalid"; + std::string builder = "invalid"; if (drv->isBuiltin()) { ; @@ -2042,13 +1964,13 @@ void LocalDerivationGoal::runChild() } args.push_back(drv->builder); } else { - builder = drv->builder.c_str(); + builder = drv->builder; args.push_back(std::string(baseNameOf(drv->builder))); } } #else else { - builder = drv->builder.c_str(); + builder = drv->builder; args.push_back(std::string(baseNameOf(drv->builder))); } #endif @@ -2075,7 +1997,7 @@ void LocalDerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); @@ -2104,9 +2026,9 @@ void LocalDerivationGoal::runChild() posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } - posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #else - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #endif throw SysError("executing '%1%'", drv->builder); @@ -2221,8 +2143,7 @@ void LocalDerivationGoal::registerOutputs() /* Pass blank Sink as we are not ready to hash data at this stage. */ NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); + auto references = scanForReferences(blank, actualPath, referenceablePaths); outputReferencesIfUnregistered.insert_or_assign( outputName, @@ -2236,8 +2157,8 @@ void LocalDerivationGoal::registerOutputs() /* Since we'll use the already installed versions of these, we can treat them as leaves and ignore any references they have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { + [&](const AlreadyRegistered &) { return StringSet {}; }, + [&](const PerhapsNeedToRegister & refs) { StringSet referencedOutputs; /* FIXME build inverted map up front so no quadratic waste here */ for (auto & r : refs.refs) @@ -2273,11 +2194,11 @@ void LocalDerivationGoal::registerOutputs() }; std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { + [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { finish(skippedFinalPath.path); return std::nullopt; }, - [&](PerhapsNeedToRegister r) -> std::optional { + [&](const PerhapsNeedToRegister & r) -> std::optional { return r.refs; }, }, outputReferencesIfUnregistered.at(outputName)); @@ -2289,7 +2210,7 @@ void LocalDerivationGoal::registerOutputs() auto rewriteOutput = [&]() { /* Apply hash rewriting if necessary. */ if (!outputRewrites.empty()) { - warn("rewriting hashes in '%1%'; cross fingers", actualPath); + debug("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: this is in-memory. */ StringSink sink; @@ -2298,10 +2219,6 @@ void LocalDerivationGoal::registerOutputs() sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); } }; @@ -2355,32 +2272,19 @@ void LocalDerivationGoal::registerOutputs() } auto got = caSink.finish().first; auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( + + auto finalPath = worker.store.makeFixedOutputPath( outputHash.method, got, outputPathName(drv->name, outputName), refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { + refs.first); + if (scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); rsink2(*sink.s); rsink2.flush(); }); @@ -2390,12 +2294,27 @@ void LocalDerivationGoal::registerOutputs() movePath(tmpPath, actualPath); } + HashResult narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { + finalPath, + narHashAndSize.first, + }; + + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + assert(newInfo0.ca); return newInfo0; }; ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { + [&](const DerivationOutputInputAddressed & output) { /* input-addressed case */ auto requiredFinalPath = output.path; /* Preemptively add rewrite rule for final hash, as that is @@ -2414,14 +2333,14 @@ void LocalDerivationGoal::registerOutputs() newInfo0.references.insert(newInfo0.path); return newInfo0; }, - [&](DerivationOutputCAFixed dof) { + [&](const DerivationOutputCAFixed & dof) { auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { .method = dof.hash.method, .hashType = dof.hash.hash.type, }); /* Check wanted hash */ - Hash & wanted = dof.hash.hash; + const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); auto got = getContentAddressHash(*newInfo0.ca); if (wanted != got) { @@ -2450,6 +2369,10 @@ void LocalDerivationGoal::registerOutputs() }, }, output.output); + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); + /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ @@ -2459,6 +2382,7 @@ void LocalDerivationGoal::registerOutputs() floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); auto optFixedPath = output.path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) @@ -2482,6 +2406,7 @@ void LocalDerivationGoal::registerOutputs() assert(newInfo.ca); } else { auto destPath = worker.store.toRealPath(finalDestPath); + deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; } @@ -2551,7 +2476,13 @@ void LocalDerivationGoal::registerOutputs() infos.emplace(outputName, std::move(newInfo)); } - if (buildMode == bmCheck) return; + if (buildMode == bmCheck) { + // In case of FOD mismatches on `--check` an error must be thrown as this is also + // a source for non-determinism. + if (delayedException) + std::rethrow_exception(delayedException); + return; + } /* Apply output checks. */ checkOutputs(infos); @@ -2636,7 +2567,7 @@ void LocalDerivationGoal::registerOutputs() that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { for (auto& [outputName, newInfo] : infos) { auto thisRealisation = Realisation{ .id = DrvOutput{initialOutputs.at(outputName).outputHash, diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index d30be2351..088a57209 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal /* Paths that were added via recursive Nix calls. */ StorePathSet addedPaths; + /* Realisations that were added via recursive Nix calls. */ + std::set addedDrvOutputs; + /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/' where @@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const DrvOutput & id) + { + return addedDrvOutputs.count(id); + } + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index e56cfadbe..29a8cfb87 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -204,7 +204,7 @@ void PathSubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), + copyStorePath(*sub, worker.store, subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0f2ade348..55afb5cca 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -9,11 +9,12 @@ namespace nix { -Worker::Worker(Store & store) +Worker::Worker(Store & store, Store & evalStore) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) , store(store) + , evalStore(evalStore) { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; @@ -238,7 +239,7 @@ void Worker::run(const Goals & _topGoals) } } - /* Call queryMissing() efficiently query substitutes. */ + /* Call queryMissing() to efficiently query substitutes. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 918de35f6..6a3b99c02 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -110,6 +110,7 @@ public: bool checkMismatch; Store & store; + Store & evalStore; std::unique_ptr hook; @@ -131,7 +132,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(Store & store); + Worker(Store & store, Store & evalStore); ~Worker(); /* Make a goal (with caching). */ diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 20ee046a1..08af0cc1f 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -3,10 +3,19 @@ -- is enabled create table if not exists Realisations ( + id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, signatures text, -- space-separated list - primary key (drvPath, outputName), foreign key (outputPath) references ValidPaths(id) on delete cascade ); + +create index if not exists IndexRealisations on Realisations(drvPath, outputName); + +create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict +); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..cf32ccdc4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -31,10 +31,10 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { - [](TextHash th) { + [](TextHash & th) { return "text:" + th.hash.to_string(Base32, true); }, - [](FixedOutputHash fsh) { + [](FixedOutputHash & fsh) { return makeFixedOutputCA(fsh.method, fsh.hash); } }, ca); @@ -43,10 +43,10 @@ std::string renderContentAddress(ContentAddress ca) std::string renderContentAddressMethod(ContentAddressMethod cam) { return std::visit(overloaded { - [](TextHashMethod &th) { + [](TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod &fshm) { + [](FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } }, cam); @@ -104,12 +104,12 @@ ContentAddress parseContentAddress(std::string_view rawCa) { return std::visit( overloaded { - [&](TextHashMethod thm) { + [&](TextHashMethod & thm) { return ContentAddress(TextHash { .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) }); }, - [&](FixedOutputHashMethod fohMethod) { + [&](FixedOutputHashMethod & fohMethod) { return ContentAddress(FixedOutputHash { .method = fohMethod.fileIngestionMethod, .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), @@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { - std::string_view asPrefix {std::string{caMethod} + ":"}; - return parseContentAddressMethodPrefix(asPrefix); + std::string asPrefix = std::string{caMethod} + ":"; + // parseContentAddressMethodPrefix takes its argument by reference + std::string_view asPrefixView = asPrefix; + return parseContentAddressMethodPrefix(asPrefixView); } std::optional parseContentAddressOpt(std::string_view rawCaOpt) @@ -137,10 +139,10 @@ std::string renderContentAddress(std::optional ca) Hash getContentAddressHash(const ContentAddress & ca) { return std::visit(overloaded { - [](TextHash th) { + [](const TextHash & th) { return th.hash; }, - [](FixedOutputHash fsh) { + [](const FixedOutputHash & fsh) { return fsh.hash; } }, ca); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0be9d2c54..2eb566080 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -227,8 +227,15 @@ struct ClientSettings try { if (name == "ssh-auth-sock") // obsolete ; + else if (name == settings.experimentalFeatures.name) { + // We don’t want to forward the experimental features to + // the daemon, as that could cause some pretty weird stuff + if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) + debug("Ignoring the client-specified experimental features"); + } else if (trusted || name == settings.buildTimeout.name + || name == settings.buildRepeat.name || name == "connect-timeout" || (name == "builders" && value == "")) settings.set(name, value); @@ -243,27 +250,10 @@ struct ClientSettings } }; -static void writeValidPathInfo( - ref store, - unsigned int clientVersion, - Sink & to, - std::shared_ptr info) -{ - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - worker_proto::write(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } -} - static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { reqs = worker_proto::read(store, from, Phantom> {}); } else { for (auto & s : readStrings(from)) @@ -406,25 +396,21 @@ static void performOp(TunnelLogger * logger, ref store, FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &_) { + [&](TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod &fohm) { - if (!refs.empty()) - throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + [&](FixedOutputHashMethod & fohm) { + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod); }(); logger->stopWork(); - to << store->printStorePath(pathInfo->path); - writeValidPathInfo(store, clientVersion, to, pathInfo); - + pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); } else { HashType hashAlgo; std::string baseName; @@ -471,6 +457,21 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopAddMultipleToStore: { + bool repair, dontCheckSigs; + from >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + + logger->startWork(); + FramedSource source(from); + store->addMultipleToStore(source, + RepairFlag{repair}, + dontCheckSigs ? NoCheckSigs : CheckSigs); + logger->stopWork(); + break; + } + case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); @@ -622,9 +623,9 @@ static void performOp(TunnelLogger * logger, ref store, break; } + // Obsolete. case wopSyncWithGC: { logger->startWork(); - store->syncWithGC(); logger->stopWork(); to << 1; break; @@ -770,7 +771,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - writeValidPathInfo(store, clientVersion, to, info); + info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -885,10 +886,15 @@ static void performOp(TunnelLogger * logger, ref store, case wopRegisterDrvOutput: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); - store->registerDrvOutput(Realisation{ - .id = outputId, .outPath = outputPath}); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + } else { + auto realisation = worker_proto::read(*store, from, Phantom()); + store->registerDrvOutput(realisation); + } logger->stopWork(); break; } @@ -898,9 +904,15 @@ static void performOp(TunnelLogger * logger, ref store, auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - std::set outPaths; - if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + } else { + std::set realisations; + if (info) realisations.insert(*info); + worker_proto::write(*store, to, realisations); + } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f6defd98f..b926bb711 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -10,18 +10,18 @@ namespace nix { std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return std::visit(overloaded { - [](DerivationOutputInputAddressed doi) -> std::optional { + [](const DerivationOutputInputAddressed & doi) -> std::optional { return { doi.path }; }, - [&](DerivationOutputCAFixed dof) -> std::optional { + [&](const DerivationOutputCAFixed & dof) -> std::optional { return { dof.path(store, drvName, outputName) }; }, - [](DerivationOutputCAFloating dof) -> std::optional { + [](const DerivationOutputCAFloating & dof) -> std::optional { return std::nullopt; }, - [](DerivationOutputDeferred) -> std::optional { + [](const DerivationOutputDeferred &) -> std::optional { return std::nullopt; }, }, output); @@ -187,7 +187,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput { .output = DerivationOutputCAFloating { @@ -332,22 +332,22 @@ string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); std::visit(overloaded { - [&](DerivationOutputInputAddressed doi) { + [&](const DerivationOutputInputAddressed & doi) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path)); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](DerivationOutputCAFixed dof) { + [&](const DerivationOutputCAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputCAFloating dof) { + [&](const DerivationOutputCAFloating & dof) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, - [&](DerivationOutputDeferred) { + [&](const DerivationOutputDeferred &) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); @@ -420,13 +420,13 @@ DerivationType BasicDerivation::type() const std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { - [&](DerivationOutputInputAddressed _) { + [&](const DerivationOutputInputAddressed &) { inputAddressedOutputs.insert(i.first); }, - [&](DerivationOutputCAFixed _) { + [&](const DerivationOutputCAFixed &) { fixedCAOutputs.insert(i.first); }, - [&](DerivationOutputCAFloating dof) { + [&](const DerivationOutputCAFloating & dof) { floatingCAOutputs.insert(i.first); if (!floatingHashType) { floatingHashType = dof.hashType; @@ -435,7 +435,7 @@ DerivationType BasicDerivation::type() const throw Error("All floating outputs must use the same hash type"); } }, - [&](DerivationOutputDeferred _) { + [&](const DerivationOutputDeferred &) { deferredIAOutputs.insert(i.first); }, }, i.second.output); @@ -538,15 +538,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation - [&](Hash drvHash) { + [&](const Hash & drvHash) { inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); }, - [&](DeferredHash deferredHash) { + [&](const DeferredHash & deferredHash) { isDeferred = true; inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); }, // CA derivation's output hashes - [&](CaOutputHashes outputHashes) { + [&](const CaOutputHashes & outputHashes) { std::set justOut = { "out" }; for (auto & output : i.second) { /* Put each one in with a single "out" output.. */ @@ -568,21 +568,21 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } -std::map staticOutputHashes(Store& store, const Derivation& drv) +std::map staticOutputHashes(Store & store, const Derivation & drv) { std::map res; std::visit(overloaded { - [&](Hash drvHash) { + [&](const Hash & drvHash) { for (auto & outputName : drv.outputNames()) { res.insert({outputName, drvHash}); } }, - [&](DeferredHash deferredHash) { + [&](const DeferredHash & deferredHash) { for (auto & outputName : drv.outputNames()) { res.insert({outputName, deferredHash.hash}); } }, - [&](CaOutputHashes outputHashes) { + [&](const CaOutputHashes & outputHashes) { res = outputHashes; }, }, hashDerivationModulo(store, drv, true)); @@ -666,22 +666,22 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr for (auto & i : drv.outputs) { out << i.first; std::visit(overloaded { - [&](DerivationOutputInputAddressed doi) { + [&](const DerivationOutputInputAddressed & doi) { out << store.printStorePath(doi.path) << "" << ""; }, - [&](DerivationOutputCAFixed dof) { + [&](const DerivationOutputCAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, - [&](DerivationOutputCAFloating dof) { + [&](const DerivationOutputCAFloating & dof) { out << "" << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, - [&](DerivationOutputDeferred) { + [&](const DerivationOutputDeferred &) { out << "" << "" << ""; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2df440536..b1cb68194 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,8 +138,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 13833c58e..3d188e981 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -11,18 +11,33 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } -nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { +nlohmann::json BuiltPath::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; + res["outputs"][output] = store->printStorePath(path); } return res; } -nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store) { +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) { auto res = nlohmann::json::array(); - for (const DerivedPathWithHints & buildable : buildables) { + for (const BuiltPath & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); }, buildable.raw()); @@ -62,7 +77,7 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi auto outputsS = s.substr(n + 1); std::set outputs; if (outputsS != "*") - outputs = tokenizeString>(outputsS); + outputs = tokenizeString>(outputsS, ","); return {drvPath, outputs}; } @@ -74,4 +89,30 @@ DerivedPath DerivedPath::parse(const Store & store, std::string_view s) : (DerivedPath) DerivedPath::Built::parse(store, s); } +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath)); + for (auto& [outputName, outputPath] : p.outputs) { + if (settings.isExperimentalFeatureEnabled( + Xp::CaDerivations)) { + auto thisRealisation = store.queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName}); + assert(thisRealisation); // We’ve built it, so we must h + // ve the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a2fe59de..9d6ace069 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "realisation.hh" #include @@ -79,51 +80,44 @@ struct DerivedPath : _DerivedPathRaw { /** * A built derived path with hints in the form of optional concrete output paths. * - * See 'DerivedPathWithHints' for more an explanation. + * See 'BuiltPath' for more an explanation. */ -struct DerivedPathWithHintsBuilt { +struct BuiltPathBuilt { StorePath drvPath; - std::map> outputs; + std::map outputs; nlohmann::json toJSON(ref store) const; - static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); + static BuiltPathBuilt parse(const Store & store, std::string_view); }; -using _DerivedPathWithHintsRaw = std::variant< +using _BuiltPathRaw = std::variant< DerivedPath::Opaque, - DerivedPathWithHintsBuilt + BuiltPathBuilt >; /** - * A derived path with hints in the form of optional concrete output paths in the built case. - * - * This type is currently just used by the CLI. The paths are filled in - * during evaluation for derivations that know what paths they will - * produce in advanced, i.e. input-addressed or fixed-output content - * addressed derivations. - * - * That isn't very good, because it puts floating content-addressed - * derivations "at a disadvantage". It would be better to never rely on - * the output path of unbuilt derivations, and exclusively use the - * realizations types to work with built derivations' concrete output - * paths. + * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * output path(s). */ -// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. -struct DerivedPathWithHints : _DerivedPathWithHintsRaw { - using Raw = _DerivedPathWithHintsRaw; +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; - using Built = DerivedPathWithHintsBuilt; + using Built = BuiltPathBuilt; inline const Raw & raw() const { return static_cast(*this); } + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + }; -typedef std::vector DerivedPathsWithHints; +typedef std::vector DerivedPaths; +typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 8f26af685..62dc21c59 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -43,11 +43,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override - { unsupported("addToStore"); } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } @@ -55,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - std::optional queryRealisation(const DrvOutput&) override - { unsupported("queryRealisation"); } + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override + { callback(nullptr); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 514ab3bf9..4621a8217 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -7,7 +7,7 @@ #include "finally.hh" #include "callback.hh" -#ifdef ENABLE_S3 +#if ENABLE_S3 #include #endif @@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); +#ifdef __linux__ + /* Cause this thread to not share any FS attributes with the main thread, + because this causes setns() in restoreMountNamespace() to fail. + Ideally, this would happen in the std::thread() constructor. */ + if (unshare(CLONE_FS) != 0) + throw SysError("unsharing filesystem state in download thread"); +#endif + std::map> items; bool quit = false; @@ -665,7 +673,7 @@ struct curlFileTransfer : public FileTransfer writeFull(wakeupPipe.writeSide.get(), " "); } -#ifdef ENABLE_S3 +#if ENABLE_S3 std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); @@ -688,7 +696,7 @@ struct curlFileTransfer : public FileTransfer if (hasPrefix(request.uri, "s3://")) { // FIXME: do this on a worker thread try { -#ifdef ENABLE_S3 +#if ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); std::string profile = get(params, "profile").value_or(""); @@ -716,15 +724,24 @@ struct curlFileTransfer : public FileTransfer } }; +ref makeCurlFileTransfer() +{ + return make_ref(); +} + ref getFileTransfer() { - static ref fileTransfer = makeFileTransfer(); + static ref fileTransfer = makeCurlFileTransfer(); + + if (fileTransfer->state_.lock()->quit) + fileTransfer = makeCurlFileTransfer(); + return fileTransfer; } ref makeFileTransfer() { - return make_ref(); + return makeCurlFileTransfer(); } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index bc692ca42..7a414da6b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -10,48 +10,22 @@ #include #include -#include -#include -#include +#include #include #include +#include +#include +#include +#include +#include +#include #include -#include namespace nix { -static string gcLockName = "gc.lock"; -static string gcRootsDir = "gcroots"; - - -/* Acquire the global GC lock. This is used to prevent new Nix - processes from starting after the temporary root files have been - read. To be precise: when they try to create a new temporary root - file, they will block until the garbage collector has finished / - yielded the GC lock. */ -AutoCloseFD LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % stateDir % gcLockName).str(); - - debug(format("acquiring global GC lock '%1%'") % fnGCLock); - - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fdGCLock) - throw SysError("opening global GC lock '%1%'", fnGCLock); - - if (!lockFile(fdGCLock.get(), lockType, false)) { - printInfo("waiting for the big garbage collector lock..."); - lockFile(fdGCLock.get(), lockType, true); - } - - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ - - return fdGCLock; -} +static std::string gcSocketPath = "/gc-socket/socket"; +static std::string gcRootsDir = "gcroots"; static void makeSymlink(const Path & link, const Path & target) @@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target) } -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); -} - - void LocalStore::addIndirectRoot(const Path & path) { string hash = hashString(htSHA1, path).to_string(Base32, false); @@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); + /* Register this root with the garbage collector, if it's + running. This should be superfluous since the caller should + have registered this root yet, but let's be on the safe + side. */ + addTempRoot(storePath); + /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) @@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - syncWithGC(); - return gcRoot; } @@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path) if (!state->fdTempRoots) { while (1) { - AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ @@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdTempRoots = openLockFile(fnTempRoots, true); - fdGCLock = -1; - - debug(format("acquiring read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); /* Check whether the garbage collector didn't get in our way. */ @@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path) } - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + if (!state->fdGCLock) + state->fdGCLock = openGCLock(); + restart: + FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + + if (!gcLock.acquired) { + /* We couldn't get a shared global GC lock, so the garbage + collector is running. So we have to connect to the garbage + collector and inform it about our root. */ + if (!state->fdRootsSocket) { + auto socketPath = stateDir.get() + gcSocketPath; + debug("connecting to '%s'", socketPath); + state->fdRootsSocket = createUnixDomainSocket(); + nix::connect(state->fdRootsSocket.get(), socketPath); + } + + try { + debug("sending GC root '%s'", printStorePath(path)); + writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + char c; + readFull(state->fdRootsSocket.get(), &c, 1); + assert(c == '1'); + debug("got ack for GC root '%s'", printStorePath(path)); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == EPIPE) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } catch (EndOfFile & e) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } + + /* Append the store path to the temporary roots file. */ string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); - - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); } static std::string censored = "{censored}"; -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); debug(format("reading temporary root file '%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); - if (!*fd) { + AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; throw SysError("opening temporary roots file '%1%'", path); } - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - /* Try to acquire a write lock without blocking. This can only succeed if the owning process has died. In that case we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { + if (lockFile(fd.get(), ltWrite, false)) { printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); - writeFull(fd->get(), "d"); + writeFull(fd.get(), "d"); continue; } - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - /* Read the entire file. */ - string contents = readFile(fd->get()); + string contents = readFile(fd.get()); /* Extract the roots. */ string::size_type pos = 0, end; @@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } - - fds.push_back(fd); /* keep open */ } } @@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor) Roots roots; findRootsNoTemp(roots, censor); - FDs fds; - findTempRoots(fds, roots, censor); + findTempRoots(roots, censor); return roots; } @@ -341,6 +324,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } +#if __linux__ static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -350,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots) throw; } } +#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { @@ -431,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #endif -#if defined(__linux__) +#if __linux__ readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); @@ -455,265 +440,139 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) struct GCLimitReached { }; -struct LocalStore::GCState -{ - const GCOptions & options; - GCResults & results; - StorePathSet roots; - StorePathSet tempRoots; - StorePathSet dead; - StorePathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - uint64_t bytesInvalidated; - bool moveToTrash = true; - bool shouldDelete; - GCState(const GCOptions & options, GCResults & results) - : options(options), results(results), bytesInvalidated(0) { } -}; - - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) -{ - return hasSuffix(path, suffix) - && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); -} - - -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - uint64_t bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; -} - - -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - uint64_t size = 0; - - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) { - StorePathSet referrers; - queryReferrers(*storePath, referrers); - for (auto & i : referrers) - if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); - size = queryPathInfo(*storePath)->narSize; - invalidatePathChecked(*storePath); - } - - Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); - - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError("getting status of %1%", realPath); - } - - printInfo(format("deleting '%1%'") % path); - - state.results.paths.insert(path); - - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (state.moveToTrash && S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - try { - if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError("making '%1%' writable", realPath); - Path tmp = trashDir + "/" + std::string(baseNameOf(path)); - if (rename(realPath.c_str(), tmp.c_str())) - throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); - state.bytesInvalidated += size; - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); - deleteGarbage(state, realPath); - } - } - } else - deleteGarbage(state, realPath); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); - } -} - - -bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) -{ - if (visited.count(path)) return false; - - if (state.alive.count(path)) return true; - - if (state.dead.count(path)) return false; - - if (state.roots.count(path)) { - debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path); - return true; - } - - visited.insert(path); - - if (!isValidPath(path)) return false; - - StorePathSet incoming; - - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); - - /* If keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) - if (maybeOutPath && - isValidPath(*maybeOutPath) && - queryPathInfo(*maybeOutPath)->deriver == path - ) - incoming.insert(*maybeOutPath); - } - - /* If keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - auto derivers = queryValidDerivers(path); - for (auto & i : derivers) - incoming.insert(i); - } - - for (auto & i : incoming) - if (i != path) - if (canReachRoot(state, visited, i)) { - state.alive.insert(path); - return true; - } - - return false; -} - - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); - if (realPath == linksDir || realPath == trashDir) return; - - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - - auto storePath = maybeParseStorePath(path); - - if (!storePath || !isValidPath(*storePath)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; - - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; - - /* Don't delete .check directories for derivations that are - currently being built, because we may need to run - diff-hook. */ - if (isActiveTempFile(state, path, ".check")) return; - } - - StorePathSet visited; - - if (storePath && canReachRoot(state, visited, *storePath)) { - debug("cannot delete '%s' because it's still reachable", path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete ‘path’ and its referrers here so that - ‘nix-store --delete’ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - for (auto & i : visited) - state.dead.insert(i); - if (state.shouldDelete) - deletePathRecursive(state, path); - } -} - - -/* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError("opening directory '%1%'", linksDir); - - int64_t actualSize = 0, unsharedSize = 0; - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; - - auto st = lstat(path); - - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; - } - - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); - - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); - - state.results.bytesFreed += st.st_size; - } - - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; - - printInfo("note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); -} - - void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - GCState state(options, results); - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool gcKeepOutputs = settings.gcKeepOutputs; + bool gcKeepDerivations = settings.gcKeepDerivations; + + StorePathSet roots, dead, alive; + + struct Shared + { + // The temp roots only store the hash part to make it easier to + // ignore suffixes like '.lock', '.chroot' and '.check'. + std::unordered_set tempRoots; + + // Hash part of the store path currently being deleted, if + // any. + std::optional pending; + }; + + Sync _shared; + + std::condition_variable wakeup; /* Using `--ignore-liveness' with `--delete' can have unintended consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs or derivers, respectively). So disable them. */ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; + gcKeepOutputs = false; + gcKeepDerivations = false; } - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - - if (state.shouldDelete) + if (shouldDelete) deletePath(reservedPath); - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + /* Acquire the global GC root. Note: we don't use fdGCLock + here because then in auto-gc mode, another thread could + downgrade our exclusive lock. */ + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + + /* Start the server for receiving new roots. */ + auto socketPath = stateDir.get() + gcSocketPath; + createDirs(dirOf(socketPath)); + auto fdServer = createUnixDomainSocket(socketPath, 0666); + + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) + throw SysError("making socket '%1%' non-blocking", socketPath); + + Pipe shutdownPipe; + shutdownPipe.create(); + + std::thread serverThread([&]() { + Sync> connections; + + Finally cleanup([&]() { + debug("GC roots server shutting down"); + while (true) { + auto item = remove_begin(*connections.lock()); + if (!item) break; + auto & [fd, thread] = *item; + shutdown(fd, SHUT_RDWR); + thread.join(); + } + }); + + while (true) { + std::vector fds; + fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN}); + fds.push_back({.fd = fdServer.get(), .events = POLLIN}); + auto count = poll(fds.data(), fds.size(), -1); + assert(count != -1); + + if (fds[0].revents) + /* Parent is asking us to quit. */ + break; + + if (fds[1].revents) { + /* Accept a new connection. */ + assert(fds[1].revents & POLLIN); + AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); + if (!fdClient) continue; + + /* Process the connection in a separate thread. */ + auto fdClient_ = fdClient.get(); + std::thread clientThread([&, fdClient = std::move(fdClient)]() { + Finally cleanup([&]() { + auto conn(connections.lock()); + auto i = conn->find(fdClient.get()); + if (i != conn->end()) { + i->second.detach(); + conn->erase(i); + } + }); + + while (true) { + try { + auto path = readLine(fdClient.get()); + auto storePath = maybeParseStorePath(path); + if (storePath) { + debug("got new GC root '%s'", path); + auto hashPart = std::string(storePath->hashPart()); + auto shared(_shared.lock()); + shared->tempRoots.insert(hashPart); + /* If this path is currently being + deleted, then we have to wait until + deletion is finished to ensure that + the client doesn't start + re-creating it before we're + done. FIXME: ideally we would use a + FD for this so we don't block the + poll loop. */ + while (shared->pending == hashPart) { + debug("synchronising with deletion of path '%s'", path); + shared.wait(wakeup); + } + } else + printError("received garbage instead of a root from client"); + writeFull(fdClient.get(), "1", false); + } catch (Error &) { break; } + } + }); + + connections.lock()->insert({fdClient_, std::move(clientThread)}); + } + } + }); + + Finally stopServer([&]() { + writeFull(shutdownPipe.writeSide.get(), "x", false); + wakeup.notify_all(); + if (serverThread.joinable()) serverThread.join(); + }); /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ @@ -722,124 +581,256 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first); + for (auto & i : rootMap) roots.insert(i.first); - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; + /* Read the temporary roots created before we acquired the global + GC root. Any new roots will be sent to our socket. */ Roots tempRoots; - findTempRoots(fds, tempRoots, true); + findTempRoots(tempRoots, true); for (auto & root : tempRoots) { - state.tempRoots.insert(root.first); - state.roots.insert(root.first); + _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); + roots.insert(root.first); } - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots' is garbage. */ + /* Helper function that deletes a path from the store and throws + GCLimitReached if we've deleted enough garbage. */ + auto deleteFromStore = [&](std::string_view baseName) + { + Path path = storeDir + "/" + std::string(baseName); + Path realPath = realStoreDir + "/" + std::string(baseName); - if (state.shouldDelete) { - if (pathExists(trashDir)) deleteGarbage(state, trashDir); - try { - createDirs(trashDir); - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo("note: can't create trash directory: %s", e.msg()); - state.moveToTrash = false; + printInfo("deleting '%1%'", path); + + results.paths.insert(path); + + uint64_t bytesFreed; + deletePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; + + if (results.bytesFreed > options.maxFreed) { + printInfo("deleted more than %d bytes; stopping", options.maxFreed); + throw GCLimitReached(); + } + }; + + std::map referrersCache; + + /* Helper function that visits all paths reachable from `start` + via the referrers edges and optionally derivers and derivation + output edges. If none of those paths are roots, then all + visited paths are garbage and are deleted. */ + auto deleteReferrersClosure = [&](const StorePath & start) { + StorePathSet visited; + std::queue todo; + + /* Wake up any GC client waiting for deletion of the paths in + 'visited' to finish. */ + Finally releasePending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); + + auto enqueue = [&](const StorePath & path) { + if (visited.insert(path).second) + todo.push(path); + }; + + enqueue(start); + + while (auto path = pop(todo)) { + checkInterrupt(); + + /* Bail out if we've previously discovered that this path + is alive. */ + if (alive.count(*path)) { + alive.insert(start); + return; + } + + /* If we've previously deleted this path, we don't have to + handle it again. */ + if (dead.count(*path)) continue; + + auto markAlive = [&]() + { + alive.insert(*path); + alive.insert(start); + try { + StorePathSet closure; + computeFSClosure(*path, closure); + for (auto & p : closure) + alive.insert(p); + } catch (InvalidPath &) { } + }; + + /* If this is a root, bail out. */ + if (roots.count(*path)) { + debug("cannot delete '%s' because it's a root", printStorePath(*path)); + return markAlive(); + } + + if (options.action == GCOptions::gcDeleteSpecific + && !options.pathsToDelete.count(*path)) + return; + + { + auto hashPart = std::string(path->hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.count(hashPart)) { + debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + return markAlive(); + } + shared->pending = hashPart; + } + + if (isValidPath(*path)) { + + /* Visit the referrers of this path. */ + auto i = referrersCache.find(*path); + if (i == referrersCache.end()) { + StorePathSet referrers; + queryReferrers(*path, referrers); + referrersCache.emplace(*path, std::move(referrers)); + i = referrersCache.find(*path); + } + for (auto & p : i->second) + enqueue(p); + + /* If keep-derivations is set and this is a + derivation, then visit the derivation outputs. */ + if (gcKeepDerivations && path->isDerivation()) { + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == *path) + enqueue(*maybeOutPath); + } + + /* If keep-outputs is set, then visit the derivers. */ + if (gcKeepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } } } - } - /* Now either delete all garbage paths, or just the specified + for (auto & path : topoSortPaths(visited)) { + if (!dead.insert(path).second) continue; + if (shouldDelete) { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); + } + } + }; + + /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC")) + readFile(*p); + + /* Either delete all garbage paths, or just the specified paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - tryToDelete(state, printStorePath(i)); - if (state.dead.find(i) == state.dead.end()) + deleteReferrersClosure(i); + if (!dead.count(i)) throw Error( - "cannot delete path '%1%' since it is still alive. " - "To find out why use: " + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " "nix-store --query --roots", printStorePath(i)); } } else if (options.maxFreed > 0) { - if (state.shouldDelete) + if (shouldDelete) printInfo("deleting garbage..."); else printInfo("determining live/dead paths..."); try { - - AutoCloseDir dir(opendir(realStoreDir.c_str())); + AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = baseNameOf(linksDir); Paths entries; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = storeDir + "/" + name; - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) - entries.push_back(path); + if (name == "." || name == ".." || name == linksName) continue; + + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + deleteReferrersClosure(*storePath); else - tryToDelete(state, path); + deleteFromStore(name); + } - - dir.reset(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector entries_(entries.begin(), entries.end()); - std::mt19937 gen(1); - std::shuffle(entries_.begin(), entries_.end(), gen); - - for (auto & i : entries_) - tryToDelete(state, i); - } catch (GCLimitReached & e) { } } - if (state.options.action == GCOptions::gcReturnLive) { - for (auto & i : state.alive) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnLive) { + for (auto & i : alive) + results.paths.insert(printStorePath(i)); return; } - if (state.options.action == GCOptions::gcReturnDead) { - for (auto & i : state.dead) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnDead) { + for (auto & i : dead) + results.paths.insert(printStorePath(i)); return; } - /* Allow other processes to add to the store from here on. */ - fdGCLock = -1; - fds.clear(); - - /* Delete the trash directory. */ - printInfo(format("deleting '%1%'") % trashDir); - deleteGarbage(state, trashDir); - - /* Clean up the links directory. */ + /* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { printInfo("deleting unused links..."); - removeUnusedLinks(state); + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError("opening directory '%1%'", linksDir); + + int64_t actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + auto st = lstat(path); + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError("deleting '%1%'", path); + + results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError("statting '%1%'", linksDir); + int64_t overhead = st.st_blocks * 512ULL; + + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } /* While we're at it, vacuum the database. */ @@ -856,7 +847,7 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.c_str(), &st)) + if (statvfs(realStoreDir.get().c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d3b27d7be..9f1a88130 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") || + pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") extraPlatforms.insert("aarch64-darwin"); else if (std::string{SYSTEM} == "aarch64-darwin") @@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const std::string & name) +bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) { auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), name) != f.end(); + return std::find(f.begin(), f.end(), feature) != f.end(); } -MissingExperimentalFeature::MissingExperimentalFeature(std::string feature) - : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature) - , missingFeature(feature) - {} - -void Settings::requireExperimentalFeature(const std::string & name) +void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) { - if (!isExperimentalFeatureEnabled(name)) - throw MissingExperimentalFeature(name); + if (!isExperimentalFeatureEnabled(feature)) + throw MissingExperimentalFeature(feature); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 6f8749254..a50eb6803 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "config.hh" #include "util.hh" +#include "experimental-features.hh" #include #include @@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -class MissingExperimentalFeature: public Error -{ -public: - std::string missingFeature; - - MissingExperimentalFeature(std::string feature); - virtual const char* sname() const override { return "MissingExperimentalFeature"; } -}; - class Settings : public Config { unsigned int getDefaultCores(); @@ -617,8 +609,10 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. The default - is `https://cache.nixos.org`. + A list of URLs of substituters, separated by whitespace. Substituters + are tried based on their Priority value, which each substituter can set + independently. Lower value means higher priority. + The default is `https://cache.nixos.org`, with a Priority of 40. )", {"binary-caches"}}; @@ -923,12 +917,12 @@ public: value. )"}; - Setting experimentalFeatures{this, {}, "experimental-features", + Setting> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; - bool isExperimentalFeatureEnabled(const std::string & name); + bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - void requireExperimentalFeature(const std::string & name); + void requireExperimentalFeature(const ExperimentalFeature &); Setting allowDirty{this, true, "allow-dirty", "Whether to allow dirty Git/Mercurial trees."}; @@ -954,6 +948,12 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting useRegistries{this, true, "use-registries", + "Whether to use flake registries to resolve flake references."}; + + Setting acceptFlakeConfig{this, false, "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 0a3afcd51..605ec4b28 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -57,8 +57,8 @@ public: { // FIXME: do this lazily? if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); - priority.setDefault(fmt("%d", cacheInfo->priority)); + wantMassQuery.setDefault(cacheInfo->wantMassQuery); + priority.setDefault(cacheInfo->priority); } else { try { BinaryCacheStore::init(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index edaf75136..4861d185e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -82,9 +82,20 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; conn->to.flush(); - unsigned int magic = readInt(conn->from); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); + StringSink saved; + try { + TeeSource tee(conn->from, saved); + unsigned int magic = readInt(tee); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + } catch (SerialisationError & e) { + /* In case the other side is waiting for our input, + close it. */ + conn->sshConn->in.close(); + auto msg = conn->from.drain(); + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(*saved.s + msg)); + } conn->remoteVersion = readInt(conn->from); if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); @@ -216,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore(const string & name, const string & s, @@ -237,6 +248,10 @@ private: conn.to << settings.buildRepeat << settings.enforceDeterminism; + + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { + conn.to << ((int) settings.keepFailed); + } } public: @@ -267,8 +282,11 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override { + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + auto conn(connections->get()); conn->to << cmdBuildPaths; @@ -276,10 +294,10 @@ public: for (auto & p : drvPaths) { auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { - [&](StorePathWithOutputs s) { + [&](const StorePathWithOutputs & s) { ss.push_back(s.to_string(*this)); }, - [&](StorePath drvPath) { + [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, }, sOrDrvPath); @@ -349,7 +367,8 @@ public: return conn->remoteVersion; } - std::optional queryRealisation(const DrvOutput&) override + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override // TODO: Implement { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 964c4017e..f93111fce 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -93,7 +93,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + realisationsPrefix); + createDirs(binaryCacheDir + "/" + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 55941b771..f8b19d00d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; + const PathSetting realStoreDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}; }; class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store @@ -34,7 +37,7 @@ public: /* Register a permanent GC root. */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); - virtual Path getRealStoreDir() { return storeDir; } + virtual Path getRealStoreDir() { return realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 83daa7506..64019314f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -53,12 +53,15 @@ struct LocalStore::State::Stmts { SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; + SQLiteStmt UpdateRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryRealisedOutput; SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; + SQLiteStmt QueryRealisationReferences; + SQLiteStmt AddRealisationReference; }; int getSchema(Path schemaPath) @@ -76,7 +79,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 1; + const int nixCASchemaVersion = 2; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) #include "ca-specific-schema.sql.gen.hh" ; db.exec(schema); + curCASchema = nixCASchemaVersion; } + + if (curCASchema < 2) { + SQLiteTxn txn(db); + // Ugly little sql dance to add a new `id` column and make it the primary key + db.exec(R"( + create table Realisations2 ( + id integer primary key autoincrement not null, + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + signatures text, -- space-separated list + foreign key (outputPath) references ValidPaths(id) on delete cascade + ); + insert into Realisations2 (drvPath, outputName, outputPath, signatures) + select drvPath, outputName, outputPath, signatures from Realisations; + drop table Realisations; + alter table Realisations2 rename to Realisations; + )"); + db.exec(R"( + create index if not exists IndexRealisations on Realisations(drvPath, outputName); + + create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict + ); + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } @@ -106,14 +141,10 @@ LocalStore::LocalStore(const Params & params) , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) - , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"} - , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) @@ -153,13 +184,13 @@ LocalStore::LocalStore(const Params & params) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; - if (stat(realStoreDir.c_str(), &st)) + if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) + if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.c_str(), perm) == -1) + if (chmod(realStoreDir.get().c_str(), perm) == -1) throw SysError("changing permissions on path '%1%'", realStoreDir); } } @@ -277,7 +308,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -307,16 +338,25 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( - insert or replace into Realisations (drvPath, outputName, outputPath, signatures) + insert into Realisations (drvPath, outputName, outputPath, signatures) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); + state->stmts->UpdateRealisedOutput.create(state->db, + R"( + update Realisations + set signatures = ? + where + drvPath = ? and + outputName = ? + ; + )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path, Realisations.signatures from Realisations + select Realisations.id, Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -328,10 +368,33 @@ LocalStore::LocalStore(const Params & params) where drvPath = ? ; )"); + state->stmts->QueryRealisationReferences.create(state->db, + R"( + select drvPath, outputName from Realisations + join RealisationsRefs on realisationReference = Realisations.id + where referrer = ?; + )"); + state->stmts->AddRealisationReference.create(state->db, + R"( + insert or replace into RealisationsRefs (referrer, realisationReference) + values ( + (select id from Realisations where drvPath = ? and outputName = ?), + (select id from Realisations where drvPath = ? and outputName = ?)); + )"); } } +AutoCloseFD LocalStore::openGCLock() +{ + Path fnGCLock = stateDir + "/gc.lock"; + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError("opening global GC lock '%1%'", fnGCLock); + return fdGCLock; +} + + LocalStore::~LocalStore() { std::shared_future future; @@ -437,14 +500,11 @@ void LocalStore::makeStoreWritable() if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) + if (statvfs(realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } #endif @@ -627,7 +687,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::optional h; for (auto & i : drv.outputs) { std::visit(overloaded { - [&](DerivationOutputInputAddressed doia) { + [&](const DerivationOutputInputAddressed & doia) { if (!h) { // somewhat expensive so we do lazily auto temp = hashDerivationModulo(*this, drv, true); @@ -639,14 +699,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); envHasRightPath(doia.path, i.first); }, - [&](DerivationOutputCAFixed dof) { + [&](const DerivationOutputCAFixed & dof) { StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); envHasRightPath(path, i.first); }, - [&](DerivationOutputCAFloating _) { + [&](const DerivationOutputCAFloating &) { /* Nothing to check */ }, - [&](DerivationOutputDeferred) { + [&](const DerivationOutputDeferred &) { }, }, i.second.output); } @@ -654,7 +714,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -663,15 +723,55 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature("ca-derivations"); - auto state(_state.lock()); + settings.requireExperimentalFeature(Xp::CaDerivations); retrySQLite([&]() { - state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) - .exec(); + auto state(_state.lock()); + if (auto oldR = queryRealisation_(*state, info.id)) { + if (info.isCompatibleWith(*oldR)) { + auto combinedSignatures = oldR->signatures; + combinedSignatures.insert(info.signatures.begin(), + info.signatures.end()); + state->stmts->UpdateRealisedOutput.use() + (concatStringsSep(" ", combinedSignatures)) + (info.id.strHash()) + (info.id.outputName) + .exec(); + } else { + throw Error("Trying to register a realisation of '%s', but we already " + "have another one locally.\n" + "Local: %s\n" + "Remote: %s", + info.id.to_string(), + printStorePath(oldR->outPath), + printStorePath(info.outPath) + ); + } + } else { + state->stmts->RegisterRealisedOutput.use() + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) + .exec(); + } + for (auto & [outputId, depPath] : info.dependentRealisations) { + auto localRealisation = queryRealisationCore_(*state, outputId); + if (!localRealisation) + throw Error("unable to register the derivation '%s' as it " + "depends on the non existent '%s'", + info.id.to_string(), outputId.to_string()); + if (localRealisation->second.outPath != depPath) + throw Error("unable to register the derivation '%s' as it " + "depends on a realisation of '%s' that doesn’t" + "match what we have locally", + info.id.to_string(), outputId.to_string()); + state->stmts->AddRealisationReference.use() + (info.id.strHash()) + (info.id.outputName) + (outputId.strHash()) + (outputId.outputName) + .exec(); + } }); } @@ -731,7 +831,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(std::string(info.path.hashPart()), + state_->pathInfoCache.upsert(std::string(info.path.to_string()), PathInfoCacheValue{ .value = std::make_shared(info) }); } @@ -909,7 +1009,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - if (!settings.isExperimentalFeatureEnabled("ca-derivations")) + if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); @@ -977,14 +1077,19 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } +// FIXME: move this, it's not specific to LocalStore. void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + auto subPath(path.first); - // recompute store path so that we can use a different store root + // Recompute store path so that we can use a different store root. if (path.second) { subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); if (sub->storeDir == storeDir) @@ -1099,7 +1204,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.hashPart())); + state_->pathInfoCache.erase(std::string(path.to_string())); } } @@ -1145,24 +1250,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); - // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !refersToSelf)) - settings.requireExperimentalFeature("ca-references"); - /* While restoring the path from the NAR, compute the hash of the NAR. */ - std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) - hashSink = std::make_unique(htSHA256); - else - hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); + HashSink hashSink(htSHA256); - TeeSource wrapperSource { source, *hashSink }; + TeeSource wrapperSource { source, hashSink }; restorePath(realPath, wrapperSource); - auto hashResult = hashSink->finish(); + auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1172,6 +1268,31 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); + if (info.ca) { + if (auto foHash = std::get_if(&*info.ca)) { + auto actualFoHash = hashCAPath( + foHash->method, + foHash->hash.type, + info.path + ); + if (foHash->hash != actualFoHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + foHash->hash.to_string(Base32, true), + actualFoHash.hash.to_string(Base32, true)); + } + } + if (auto textHash = std::get_if(&*info.ca)) { + auto actualTextHash = hashString(htSHA256, readFile(realPath)); + if (textHash->hash != actualTextHash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + textHash->hash.to_string(Base32, true), + actualTextHash.to_string(Base32, true)); + } + } + } + autoGC(); canonicalisePathMetaData(realPath, -1); @@ -1187,7 +1308,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1243,7 +1364,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto dstPath = makeFixedOutputPath(method, hash, name, references); addTempRoot(dstPath); @@ -1290,6 +1411,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, ValidPathInfo info { dstPath, narHash.first }; info.narSize = narHash.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1390,7 +1512,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); @@ -1401,8 +1524,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; PathSet done; - fdGCLock = -1; - for (auto & i : queryAllValidPaths()) verifyPath(printStorePath(i), store, done, validPaths, repair, errors); @@ -1440,14 +1561,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - dumpPath(Store::toRealPath(i), *hashSink); - auto current = hashSink->finish(); + dumpPath(Store::toRealPath(i), hashSink); + auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", @@ -1665,19 +1782,108 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryRealisation( - const DrvOutput& id) { - typedef std::optional Ret; - return retrySQLite([&]() -> Ret { - auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( - id.outputName)); - if (!use.next()) - return std::nullopt; - auto outputPath = parseStorePath(use.getStr(0)); - auto signatures = tokenizeString(use.getStr(1)); - return Ret{Realisation{ - .id = id, .outPath = outputPath, .signatures = signatures}}; - }); +std::optional> LocalStore::queryRealisationCore_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto useQueryRealisedOutput( + state.stmts->QueryRealisedOutput.use() + (id.strHash()) + (id.outputName)); + if (!useQueryRealisedOutput.next()) + return std::nullopt; + auto realisationDbId = useQueryRealisedOutput.getInt(0); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto signatures = + tokenizeString(useQueryRealisedOutput.getStr(2)); + + return {{ + realisationDbId, + Realisation{ + .id = id, + .outPath = outputPath, + .signatures = signatures, + } + }}; } + +std::optional LocalStore::queryRealisation_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto maybeCore = queryRealisationCore_(state, id); + if (!maybeCore) + return std::nullopt; + auto [realisationDbId, res] = *maybeCore; + + std::map dependentRealisations; + auto useRealisationRefs( + state.stmts->QueryRealisationReferences.use() + (realisationDbId)); + while (useRealisationRefs.next()) { + auto depId = DrvOutput { + Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), + useRealisationRefs.getStr(1), + }; + auto dependentRealisation = queryRealisationCore_(state, depId); + assert(dependentRealisation); // Enforced by the db schema + auto outputPath = dependentRealisation->second.outPath; + dependentRealisations.insert({depId, outputPath}); + } + + res.dependentRealisations = dependentRealisations; + + return { res }; +} + +void LocalStore::queryRealisationUncached(const DrvOutput & id, + Callback> callback) noexcept +{ + try { + auto maybeRealisation + = retrySQLite>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); + if (maybeRealisation) + callback( + std::make_shared(maybeRealisation.value())); + else + callback(nullptr); + + } catch (...) { + callback.rethrow(); + } +} + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, const HashType & hashType, + const StorePath & path) +{ + return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); +} + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash +) +{ + HashModuloSink caSink ( hashType, std::string(pathHash) ); + switch (method) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + auto hash = caSink.finish().first; + return FixedOutputHash{ + .method = method, + .hash = hash, + }; +} + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 26e034a82..115ea046a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,9 +58,15 @@ private: struct Stmts; std::unique_ptr stmts; + /* The global GC lock */ + AutoCloseFD fdGCLock; + /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + /* Connection to the garbage collector. */ + AutoCloseFD fdRootsSocket; + /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point lastGCCheck; @@ -83,14 +89,10 @@ private: public: - PathSetting realStoreDir_; - - const Path realStoreDir; const Path dbDir; const Path linksDir; const Path reservedPath; const Path schemaPath; - const Path trashDir; const Path tempRootsDir; const Path fnTempRoots; @@ -143,7 +145,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -152,14 +154,11 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - private: - typedef std::shared_ptr FDPtr; - typedef list FDs; + void findTempRoots(Roots & roots, bool censor); - void findTempRoots(FDs & fds, Roots & roots, bool censor); + AutoCloseFD openGCLock(); public: @@ -206,7 +205,10 @@ public: void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); - std::optional queryRealisation(const DrvOutput&) override; + std::optional queryRealisation_(State & state, const DrvOutput & id); + std::optional> queryRealisationCore_(State & state, const DrvOutput & id); + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; private: @@ -237,29 +239,12 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - struct GCState; - - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - AutoCloseFD openGCLock(LockType lockType); - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - void removeUnusedLinks(const GCState & state); - Path createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); @@ -279,10 +264,21 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - Path getRealStoreDir() override { return realStoreDir; } - void createUser(const std::string & userName, uid_t userId) override; + // XXX: Make a generic `Store` method + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const StorePath & path); + + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash + ); + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index cf0933705..b992bcbc0 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,12 +8,12 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil -libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifneq ($(OS), FreeBSD) +libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb endif @@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1) libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif -ifeq ($(OS), SunOS) +ifdef HOST_SOLARIS libstore_LDFLAGS += -lsocket endif @@ -60,7 +60,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index b42e5e434..b6270a81b 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : storeUri( - // Backwards compatibility: if the URI is a hostname, - // prepend ssh://. + // Backwards compatibility: if the URI is schemeless, is not a path, + // and is not one of the special store connection words, prepend + // ssh://. storeUri.find("://") != std::string::npos - || hasPrefix(storeUri, "local") - || hasPrefix(storeUri, "remote") - || hasPrefix(storeUri, "auto") - || hasPrefix(storeUri, "/") + || storeUri.find("/") != std::string::npos + || storeUri == "auto" + || storeUri == "daemon" + || storeUri == "local" + || hasPrefix(storeUri, "auto?") + || hasPrefix(storeUri, "daemon?") + || hasPrefix(storeUri, "local?") + || hasPrefix(storeUri, "?") ? storeUri : "ssh://" + storeUri), systemTypes(systemTypes), @@ -34,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri, sshPublicHostKey(sshPublicHostKey) {} -bool Machine::allSupported(const std::set & features) const { +bool Machine::allSupported(const std::set & features) const +{ return std::all_of(features.begin(), features.end(), [&](const string & feature) { return supportedFeatures.count(feature) || @@ -42,14 +48,16 @@ bool Machine::allSupported(const std::set & features) const { }); } -bool Machine::mandatoryMet(const std::set & features) const { +bool Machine::mandatoryMet(const std::set & features) const +{ return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), [&](const string & feature) { return features.count(feature); }); } -ref Machine::openStore() const { +ref Machine::openStore() const +{ Store::Params storeParams; if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; @@ -78,53 +86,86 @@ ref Machine::openStore() const { return nix::openStore(storeUri, storeParams); } -void parseMachines(const std::string & s, Machines & machines) +static std::vector expandBuilderLines(const std::string & builders) { - for (auto line : tokenizeString>(s, "\n;")) { + std::vector result; + for (auto line : tokenizeString>(builders, "\n;")) { trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; if (line[0] == '@') { - auto file = trim(std::string(line, 1)); + const std::string path = trim(std::string(line, 1)); + std::string text; try { - parseMachines(readFile(file), machines); + text = readFile(path); } catch (const SysError & e) { if (e.errNo != ENOENT) throw; - debug("cannot find machines file '%s'", file); + debug("cannot find machines file '%s'", path); } + + const auto lines = expandBuilderLines(text); + result.insert(end(result), begin(lines), end(lines)); continue; } - auto tokens = tokenizeString>(line); - auto sz = tokens.size(); - if (sz < 1) - throw FormatError("bad machine specification '%s'", line); - - auto isSet = [&](size_t n) { - return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; - }; - - machines.emplace_back(tokens[0], - isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, - isSet(2) ? tokens[2] : "", - isSet(3) ? std::stoull(tokens[3]) : 1LL, - isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, - isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, - isSet(7) ? tokens[7] : ""); + result.emplace_back(line); } + return result; +} + +static Machine parseBuilderLine(const std::string & line) +{ + const auto tokens = tokenizeString>(line); + + auto isSet = [&](size_t fieldIndex) { + return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-"; + }; + + auto parseUnsignedIntField = [&](size_t fieldIndex) { + const auto result = string2Int(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line); + } + return result.value(); + }; + + auto ensureBase64 = [&](size_t fieldIndex) { + const auto & str = tokens[fieldIndex]; + try { + base64Decode(str); + } catch (const Error & e) { + throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what()); + } + return str; + }; + + if (!isSet(0)) + throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + + return { + tokens[0], + isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? parseUnsignedIntField(3) : 1U, + isSet(4) ? parseUnsignedIntField(4) : 1U, + isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + isSet(7) ? ensureBase64(7) : "" + }; +} + +static Machines parseBuilderLines(const std::vector& builders) { + Machines result; + std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + return result; } Machines getMachines() { - static auto machines = [&]() { - Machines machines; - parseMachines(settings.builders, machines); - return machines; - }(); - return machines; + const auto builderLines = expandBuilderLines(settings.builders); + return parseBuilderLines(builderLines); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a99a2fc78..32786e963 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -6,98 +6,73 @@ #include "thread-pool.hh" #include "topo-sort.hh" #include "callback.hh" +#include "closure.hh" namespace nix { - void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - struct State - { - size_t pending; - StorePathSet & paths; - std::exception_ptr exc; - }; + std::function(const StorePath & path, std::future> &)> queryDeps; + if (flipDirection) + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + StorePathSet referrers; + queryReferrers(path, referrers); + for (auto& ref : referrers) + if (ref != path) + res.insert(ref); - Sync state_(State{0, paths_, 0}); + if (includeOutputs) + for (auto& i : queryValidDerivers(path)) + res.insert(i); - std::function enqueue; + if (includeDerivers && path.isDerivation()) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); + return res; + }; + else + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + auto info = fut.get(); + for (auto& ref : info->references) + if (ref != path) + res.insert(ref); - std::condition_variable done; + if (includeOutputs && path.isDerivation()) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); - enqueue = [&](const StorePath & path) -> void { - { - auto state(state_.lock()); - if (state->exc) return; - if (!state->paths.insert(path).second) return; - state->pending++; - } + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + res.insert(*info->deriver); + return res; + }; - queryPathInfo(path, {[&](std::future> fut) { - // FIXME: calls to isValidPath() should be async - - try { - auto info = fut.get(); - - if (flipDirection) { - - StorePathSet referrers; - queryReferrers(path, referrers); - for (auto & ref : referrers) - if (ref != path) - enqueue(ref); - - if (includeOutputs) - for (auto & i : queryValidDerivers(path)) - enqueue(i); - - if (includeDerivers && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); - - } else { - - for (auto & ref : info->references) - if (ref != path) - enqueue(ref); - - if (includeOutputs && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); - - if (includeDerivers && info->deriver && isValidPath(*info->deriver)) - enqueue(*info->deriver); - - } - - { - auto state(state_.lock()); - assert(state->pending); - if (!--state->pending) done.notify_one(); - } - - } catch (...) { - auto state(state_.lock()); - if (!state->exc) state->exc = std::current_exception(); - assert(state->pending); - if (!--state->pending) done.notify_one(); - }; - }}); - }; - - for (auto & startPath : startPaths) - enqueue(startPath); - - { - auto state(state_.lock()); - while (state->pending) state.wait(done); - if (state->exc) std::rethrow_exception(state->exc); - } + computeClosure( + startPaths, paths_, + [&](const StorePath& path, + std::function>&)> + processEdges) { + std::promise> promise; + std::function>)> + getDependencies = + [&](std::future> fut) { + try { + promise.set_value(queryDeps(path, fut)); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }; + queryPathInfo(path, getDependencies); + processEdges(promise); + }); } - void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { @@ -191,7 +166,7 @@ void Store::queryMissing(const std::vector & targets, } std::visit(overloaded { - [&](DerivedPath::Built bfd) { + [&](const DerivedPath::Built & bfd) { if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); @@ -224,7 +199,7 @@ void Store::queryMissing(const std::vector & targets, mustBuildDrv(bfd.drvPath, *drv); }, - [&](DerivedPath::Opaque bo) { + [&](const DerivedPath::Opaque & bo) { if (isValidPath(bo.path)) return; @@ -264,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) { return topoSort(paths, {[&](const StorePath & path) { - StorePathSet references; try { - references = queryPathInfo(path)->references; + return queryPathInfo(path)->references; } catch (InvalidPath &) { + return StorePathSet(); } - return references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -279,5 +253,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } +std::map drvOutputReferences( + const std::set & inputRealisations, + const StorePathSet & pathReferences) +{ + std::map res; + for (const auto & input : inputRealisations) { + if (pathReferences.count(input.outPath)) { + res.insert({input.id, input.outPath}); + } + } + + return res; +} + +std::map drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath) +{ + std::set inputRealisations; + + for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto& outputName : outputNames) { + auto thisRealisation = store.queryRealisation( + DrvOutput{outputHashes.at(outputName), outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } + } + + auto info = store.queryPathInfo(outputPath); + + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); +} } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1d8d2d57e..9dd81ddfb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include +#include namespace nix { @@ -38,6 +39,15 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); +create table if not exists Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or null if the realisation is absent + timestamp integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + create table if not exists LastPurge ( dummy text primary key, value integer @@ -63,7 +73,9 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, + queryNAR, insertRealisation, insertMissingRealisation, + queryRealisation, purgeCache; std::map caches; }; @@ -98,6 +110,26 @@ public: state->queryNAR.create(state->db, "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + state->insertRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, content, timestamp) + values (?, ?, ?, ?) + )"); + + state->insertMissingRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, timestamp) + values (?, ?, ?) + )"); + + state->queryRealisation.create(state->db, + R"( + select content from Realisations + where cache = ? and outputId = ? and + ((content is null and timestamp > ?) or + (content is not null and timestamp > ?)) + )"); + /* Periodically purge expired entries from the database. */ retrySQLite([&]() { auto now = time(0); @@ -212,6 +244,38 @@ public: }); } + std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) override + { + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryRealisation(state->queryRealisation.use() + (cache.id) + (id.to_string()) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryRealisation.next()) + return {oUnknown, 0}; + + if (queryRealisation.isNull(0)) + return {oInvalid, 0}; + + auto realisation = + std::make_shared(Realisation::fromJSON( + nlohmann::json::parse(queryRealisation.getStr(0)), + "Local disk cache")); + + return {oValid, realisation}; + }); + } + void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) override @@ -251,6 +315,39 @@ public: } }); } + + void upsertRealisation( + const std::string & uri, + const Realisation & realisation) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + state->insertRealisation.use() + (cache.id) + (realisation.id.to_string()) + (realisation.toJSON().dump()) + (time(0)).exec(); + }); + + } + + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + state->insertMissingRealisation.use() + (cache.id) + (id.to_string()) + (time(0)).exec(); + }); + } }; ref getNarInfoDiskCache() diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 04de2c5eb..2dcaa76a4 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -2,6 +2,7 @@ #include "ref.hh" #include "nar-info.hh" +#include "realisation.hh" namespace nix { @@ -29,6 +30,15 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) = 0; + + virtual void upsertRealisation( + const std::string & uri, + const Realisation & realisation) = 0; + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) = 0; + virtual std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 78d587139..d95e54af1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ - bool mustToggle = dirOf(path) != realStoreDir; + bool mustToggle = dirOf(path) != realStoreDir.get(); if (mustToggle) makeWritable(dirOf(path)); /* When we're done, make the directory read-only again and reset diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c5c3ae3dc..caddba9b1 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,6 +1,8 @@ #include "parsed-derivations.hh" #include +#include +#include "json.hh" namespace nix { @@ -91,6 +93,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); + if (!derivationHasKnownOutputPaths(drv.type())) + res.insert("ca-derivations"); return res; } @@ -121,4 +125,107 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + +std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) +{ + auto structuredAttrs = getStructuredAttrs(); + if (!structuredAttrs) return std::nullopt; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv.outputs) + outputs[i.first] = hashPlaceholder(i.first); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get())); + store.pathInfoToJSON(jsonRoot, + store.exportReferences(storePaths, inputPaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + return json; +} + +/* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ +std::string writeStructuredAttrsShell(const nlohmann::json & json) +{ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto & [key, value] : json.items()) { + + if (!std::regex_match(key, shVarName)) continue; + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", key, *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto & value2 : value) { + auto s3 = handleSimpleType(value2); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", key, s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto & [key2, value2] : value.items()) { + auto s3 = handleSimpleType(value2); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(key2), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", key, s2); + } + } + + return jsonSh; +} } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index c9fbe68c4..effcf099d 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -36,6 +36,10 @@ public: bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; + + std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; +std::string writeStructuredAttrsShell(const nlohmann::json & json); + } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc new file mode 100644 index 000000000..fda55b2b6 --- /dev/null +++ b/src/libstore/path-info.cc @@ -0,0 +1,46 @@ +#include "path-info.hh" +#include "worker-protocol.hh" + +namespace nix { + +ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) +{ + return read(source, store, format, store.parseStorePath(readString(source))); +} + +ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) +{ + auto deriver = readString(source); + auto narHash = Hash::parseAny(readString(source), htSHA256); + ValidPathInfo info(path, narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = worker_proto::read(store, source, Phantom {}); + source >> info.registrationTime >> info.narSize; + if (format >= 16) { + source >> info.ultimate; + info.sigs = readStrings(source); + info.ca = parseContentAddressOpt(readString(source)); + } + return info; +} + +void ValidPathInfo::write( + Sink & sink, + const Store & store, + unsigned int format, + bool includePath) const +{ + if (includePath) + sink << store.printStorePath(path); + sink << (deriver ? store.printStorePath(*deriver) : "") + << narHash.to_string(Base16, false); + worker_proto::write(store, sink, references); + sink << registrationTime << narSize; + if (format >= 16) { + sink << ultimate + << sigs + << renderContentAddress(ca); + } +} + +} diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index de87f8b33..b4b54e593 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -105,6 +105,11 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; virtual ~ValidPathInfo() { } + + static ValidPathInfo read(Source & source, const Store & store, unsigned int format); + static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); + + void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; typedef std::map ValidPathInfos; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 865d64cf2..e5a121e00 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -31,14 +31,14 @@ std::vector toDerivedPaths(const std::vector std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](DerivedPath::Opaque bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> std::variant { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](DerivedPath::Built bfd) -> std::variant { + [&](const DerivedPath::Built & bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, }, p.raw()); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 926f4ea1e..2da74e262 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths) } +FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) + : fd(fd) +{ + if (wait) { + if (!lockFile(fd, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(fd, lockType, true); + } + } else + acquired = lockFile(fd, lockType, false); +} + + } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 411da0222..919c8904c 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -35,4 +35,18 @@ public: void setDeletion(bool deletePaths); }; +struct FdLock +{ + int fd; + bool acquired = false; + + FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + + ~FdLock() + { + if (acquired) + lockFile(fd, ltNone, false); + } +}; + } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 5d1723886..73163424c 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -126,9 +126,9 @@ void deleteGeneration(const Path & profile, GenerationNumber gen) static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) - printInfo(format("would remove generation %1%") % gen); + notice("would remove profile version %1%", gen); else { - printInfo(format("removing generation %1%") % gen); + notice("removing profile version %1%", gen); deleteGeneration(profile, gen); } } @@ -142,7 +142,7 @@ void deleteGenerations(const Path & profile, const std::set & auto [gens, curGen] = findGenerations(profile); if (gensToDelete.count(*curGen)) - throw Error("cannot delete current generation of profile %1%'", profile); + throw Error("cannot delete current version of profile %1%'", profile); for (auto & i : gens) { if (!gensToDelete.count(i.number)) continue; @@ -211,12 +211,15 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) { + if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') + throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); + time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); auto days = string2Int(strDays); if (!days || *days < 1) - throw Error("invalid number of days specifier '%1%'", timeSpec); + throw UsageError("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - *days * 24 * 3600; @@ -233,6 +236,37 @@ void switchLink(Path link, Path target) } +void switchGeneration( + const Path & profile, + std::optional dstGen, + bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + auto [gens, curGen] = findGenerations(profile); + + std::optional dst; + for (auto & i : gens) + if ((!dstGen && i.number < curGen) || + (dstGen && i.number == *dstGen)) + dst = i; + + if (!dst) { + if (dstGen) + throw Error("profile version %1% does not exist", *dstGen); + else + throw Error("no profile version older than the current (%1%) exists", curGen.value_or(0)); + } + + notice("switching profile from version %d to %d", curGen.value_or(0), dst->number); + + if (dryRun) return; + + switchLink(profile, dst->path); +} + + void lockProfile(PathLocks & lock, const Path & profile) { lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index be55a65d4..d100c970c 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -11,7 +11,7 @@ namespace nix { class StorePath; -typedef unsigned int GenerationNumber; +typedef uint64_t GenerationNumber; struct Generation { @@ -46,6 +46,13 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b void switchLink(Path link, Path target); +/* Roll back a profile to the specified generation, or to the most + recent one older than the current. */ +void switchGeneration( + const Path & profile, + std::optional dstGen, + bool dryRun); + /* Ensure exclusive access to a profile. Any command that modifies the profile first acquires this lock. */ void lockProfile(PathLocks & lock, const Path & profile); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 638065547..f871e6437 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include "closure.hh" #include namespace nix { @@ -21,11 +22,52 @@ std::string DrvOutput::to_string() const { return strHash() + "!" + outputName; } +std::set Realisation::closure(Store & store, const std::set & startOutputs) +{ + std::set res; + Realisation::closure(store, startOutputs, res); + return res; +} + +void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) +{ + auto getDeps = [&](const Realisation& current) -> std::set { + std::set res; + for (auto& [currentDep, _] : current.dependentRealisations) { + if (auto currentRealisation = store.queryRealisation(currentDep)) + res.insert(*currentRealisation); + else + throw Error( + "Unrealised derivation '%s'", currentDep.to_string()); + } + return res; + }; + + computeClosure( + startOutputs, res, + [&](const Realisation& current, + std::function>&)> + processEdges) { + std::promise> promise; + try { + auto res = getDeps(current); + promise.set_value(res); + } catch (...) { + promise.set_exception(std::current_exception()); + } + return processEdges(promise); + }); +} + nlohmann::json Realisation::toJSON() const { + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, {"signatures", signatures}, + {"dependentRealisations", jsonDependentRealisations}, }; } @@ -51,10 +93,16 @@ Realisation Realisation::fromJSON( if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + std::map dependentRealisations; + if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) + for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get>()) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), .signatures = signatures, + .dependentRealisations = dependentRealisations, }; } @@ -92,6 +140,24 @@ StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } +bool Realisation::isCompatibleWith(const Realisation & other) const +{ + assert (id == other.id); + if (outPath == other.outPath) { + if (dependentRealisations.empty() != other.dependentRealisations.empty()) { + warn( + "Encountered a realisation for '%s' with an empty set of " + "dependencies. This is likely an artifact from an older Nix. " + "I’ll try to fix the realisation if I can", + id.to_string()); + return true; + } else if (dependentRealisations == other.dependentRealisations) { + return true; + } + } + return false; +} + void RealisedPath::closure( Store& store, const RealisedPath::Set& startPaths, diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f5049c9e9..9070a6ee2 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -28,6 +28,14 @@ struct Realisation { StringSet signatures; + /** + * The realisations that are required for the current one to be valid. + * + * When importing this realisation, the store will first check that all its + * dependencies exist, and map to the correct output path + */ + std::map dependentRealisations; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); @@ -36,6 +44,11 @@ struct Realisation { bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; + static std::set closure(Store &, const std::set &); + static void closure(Store &, const std::set &, std::set & res); + + bool isCompatibleWith(const Realisation & other) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 39c4970c6..91b3fc142 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -5,27 +5,29 @@ #include #include +#include namespace nix { -static unsigned int refLength = 32; /* characters */ +static size_t refLength = 32; /* characters */ -static void search(const unsigned char * s, size_t len, - StringSet & hashes, StringSet & seen) +static void search( + std::string_view s, + StringSet & hashes, + StringSet & seen) { - static bool initialised = false; + static std::once_flag initialised; static bool isBase32[256]; - if (!initialised) { + std::call_once(initialised, [](){ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; for (unsigned int i = 0; i < base32Chars.size(); ++i) isBase32[(unsigned char) base32Chars[i]] = true; - initialised = true; - } + }); - for (size_t i = 0; i + refLength <= len; ) { + for (size_t i = 0; i + refLength <= s.size(); ) { int j; bool match = true; for (j = refLength - 1; j >= 0; --j) @@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len, break; } if (!match) continue; - string ref((const char *) s + i, refLength); + std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i); @@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len, } -struct RefScanSink : Sink +void RefScanSink::operator () (std::string_view data) { - StringSet hashes; - StringSet seen; + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + auto s = tail; + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); + search(s, hashes, seen); - string tail; + search(data, hashes, seen); - RefScanSink() { } - - void operator () (std::string_view data) override - { - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + std::string(data, 0, refLength); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search((const unsigned char *) data.data(), data.size(), hashes, seen); - - size_t tailLen = data.size() <= refLength ? data.size() : refLength; - tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)); - tail.append({data.data() + data.size() - tailLen, tailLen}); - } -}; + auto rest = refLength - tailLen; + if (rest < tail.size()) + tail = tail.substr(tail.size() - rest); + tail.append(data.data() + data.size() - tailLen, tailLen); +} -std::pair scanForReferences(const string & path, - const PathSet & refs) +std::pair scanForReferences( + const string & path, + const StorePathSet & refs) { HashSink hashSink { htSHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); - return std::pair(found, hash); + return std::pair(found, hash); } -PathSet scanForReferences(Sink & toTee, - const string & path, const PathSet & refs) +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) { - RefScanSink refsSink; - TeeSink sink { refsSink, toTee }; - std::map backMap; + StringSet hashes; + std::map backMap; for (auto & i : refs) { - auto baseName = std::string(baseNameOf(i)); - string::size_type pos = baseName.find('-'); - if (pos == string::npos) - throw Error("bad reference '%1%'", i); - string s = string(baseName, 0, pos); - assert(s.size() == refLength); - assert(backMap.find(s) == backMap.end()); - // parseHash(htSHA256, s); - refsSink.hashes.insert(s); - backMap[s] = i; + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); } /* Look for the hashes in the NAR dump of the path. */ + RefScanSink refsSink(std::move(hashes)); + TeeSink sink { refsSink, toTee }; dumpPath(path, sink); /* Map the hashes found back to their store paths. */ - PathSet found; - for (auto & i : refsSink.seen) { - std::map::iterator j; - if ((j = backMap.find(i)) == backMap.end()) abort(); + StorePathSet found; + for (auto & i : refsSink.getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); found.insert(j->second); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 4f12e6b21..a6119c861 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,13 +1,31 @@ #pragma once -#include "types.hh" #include "hash.hh" +#include "path.hh" namespace nix { -std::pair scanForReferences(const Path & path, const PathSet & refs); +std::pair scanForReferences(const Path & path, const StorePathSet & refs); -PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs); +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class RefScanSink : public Sink +{ + StringSet hashes; + StringSet seen; + + std::string tail; + +public: + + RefScanSink(StringSet && hashes) : hashes(hashes) + { } + + StringSet & getResult() + { return seen; } + + void operator () (std::string_view data) override; +}; struct RewritingSink : Sink { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 761b4a087..a627e9cf1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -162,8 +162,19 @@ void RemoteStore::initConnection(Connection & conn) try { conn.to << WORKER_MAGIC_1; conn.to.flush(); - unsigned int magic = readInt(conn.from); - if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + StringSink saved; + try { + TeeSource tee(conn.from, saved); + unsigned int magic = readInt(tee); + if (magic != WORKER_MAGIC_2) + throw Error("protocol mismatch"); + } catch (SerialisationError & e) { + /* In case the other side is waiting for our input, close + it. */ + conn.closeWrite(); + auto msg = conn.from.drain(); + throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg)); + } conn.from >> conn.daemonVersion; if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) @@ -222,6 +233,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); + overrides.erase(settings.experimentalFeatures.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -278,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection() return ConnectionHandle(connections->get()); } +void RemoteStore::setOptions() +{ + setOptions(*(getConnection().handle)); +} bool RemoteStore::isValidPathUncached(const StorePath & path) { @@ -386,23 +402,6 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } -ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) -{ - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - auto info = make_ref(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } - return info; -} - - void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { @@ -423,7 +422,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = readValidPathInfo(conn, path); + info = std::make_shared( + ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -525,20 +525,20 @@ ref RemoteStore::addCAToStore( }); } - auto path = parseStorePath(readString(conn->from)); - return readValidPathInfo(conn, path); + return make_ref( + ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); std::visit(overloaded { - [&](TextHashMethod thm) -> void { + [&](const TextHashMethod & thm) -> void { std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; worker_proto::write(*this, conn->to, references); conn.processStderr(); }, - [&](FixedOutputHashMethod fohm) -> void { + [&](const FixedOutputHashMethod & fohm) -> void { conn->to << wopAddToStore << name @@ -582,9 +582,8 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) + FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - StorePathSet references; return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } @@ -642,6 +641,25 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + auto conn(getConnection()); + conn->to + << wopAddMultipleToStore + << repair + << !checkSigs; + conn.withFramedSink([&](Sink & sink) { + source.drainInto(sink); + }); + } else + Store::addMultipleToStore(source, repair, checkSigs); +} + + StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { @@ -653,36 +671,57 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << wopRegisterDrvOutput; - conn->to << info.id.to_string(); - conn->to << std::string(info.outPath.to_string()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + } else { + worker_proto::write(*this, conn->to, info); + } conn.processStderr(); } -std::optional RemoteStore::queryRealisation(const DrvOutput & id) +void RemoteStore::queryRealisationUncached(const DrvOutput & id, + Callback> callback) noexcept { auto conn(getConnection()); conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + + auto real = [&]() -> std::shared_ptr { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared(*realisations.begin()); + } + }(); + + try { + callback(std::shared_ptr(real)); + } catch (...) { return callback.rethrow(); } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { worker_proto::write(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { - [&](StorePathWithOutputs s) { + [&](const StorePathWithOutputs & s) { ss.push_back(s.to_string(store)); }, - [&](StorePath drvPath) { + [&](const StorePath & drvPath) { throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), GET_PROTOCOL_MAJOR(conn->daemonVersion), @@ -694,8 +733,18 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) { + if (evalStore && evalStore.get() != this) { + /* The remote doesn't have a way to access evalStore, so copy + the .drvs. */ + RealisedPath::Set drvPaths2; + for (auto & i : drvPaths) + if (auto p = std::get_if(&i)) + drvPaths2.insert(p->drvPath); + copyClosure(*evalStore, *this, drvPaths2); + } + auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); @@ -761,15 +810,6 @@ void RemoteStore::addIndirectRoot(const Path & path) } -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); @@ -990,14 +1030,14 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); std::exception_ptr ex; - /* Handle log messages / exceptions from the remote on a - separate thread. */ + /* Handle log messages / exceptions from the remote on a separate + thread. */ std::thread stderrThread([&]() { try { @@ -1030,7 +1070,6 @@ void ConnectionHandle::withFramedSink(std::function fun) stderrThread.join(); if (ex) std::rethrow_exception(ex); - } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6cf76a46d..0fd67f371 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -73,19 +73,25 @@ public: /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -96,8 +102,6 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -120,7 +124,6 @@ public: struct Connection { - AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; @@ -128,6 +131,8 @@ public: virtual ~Connection(); + virtual void closeWrite() = 0; + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; @@ -143,6 +148,8 @@ protected: virtual void setOptions(Connection & conn); + void setOptions() override; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -151,8 +158,6 @@ protected: virtual void narFromPath(const StorePath & path, Sink & sink) override; - ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); - private: std::atomic_bool failed{false}; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6bfbee044..7accad7f4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -209,7 +209,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3Helper s3Helper; S3BinaryCacheStoreImpl( - const std::string & scheme, + const std::string & uriScheme, const std::string & bucketName, const Params & params) : StoreConfig(params) @@ -232,8 +232,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { if (auto cacheInfo = diskCache->cacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); - priority.setDefault(fmt("%d", cacheInfo->priority)); + wantMassQuery.setDefault(cacheInfo->wantMassQuery); + priority.setDefault(cacheInfo->priority); } else { BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 351037822..56b35c3fe 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -32,7 +32,9 @@ (literal "/tmp") (subpath TMPDIR)) ; Some packages like to read the system version. -(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) ; Without this line clang cannot write to /dev/null, breaking some configure tests. (allow file-read-metadata (literal "/dev")) @@ -95,3 +97,8 @@ ; This is used by /bin/sh on macOS 10.15 and later. (allow file* (literal "/private/var/select/sh")) + +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. +(allow file-read* + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah")) diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 02d0810cc..3f76baa82 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 6) +#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 447b4179b..1d6baf02d 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path, bool create) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index f2caf2aeb..bb03daef4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -57,6 +57,11 @@ private: struct Connection : RemoteStore::Connection { std::unique_ptr sshConn; + + void closeWrite() override + { + sshConn->in.close(); + } }; ref openConnection() override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 93fcb068f..c88dfe179 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -9,6 +9,7 @@ #include "url.hh" #include "archive.hh" #include "callback.hh" +#include "remote-store.hh" #include @@ -198,10 +199,10 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress { // New template return std::visit(overloaded { - [&](TextHash th) { + [&](const TextHash & th) { return makeTextPath(name, th.hash, references); }, - [&](FixedOutputHash fsh) { + [&](const FixedOutputHash & fsh) { return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); } }, ca); @@ -236,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -245,7 +246,21 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, else readFile(srcPath, sink); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); + return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); +} + + +void Store::addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto expected = readNum(source); + for (uint64_t i = 0; i < expected; ++i) { + auto info = ValidPathInfo::read(source, *this, 16); + info.ultimate = false; + addToStore(info, source, repair, checkSigs); + } } @@ -337,6 +352,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, return info; } +StringSet StoreConfig::getDefaultSystemFeatures() +{ + auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + res.insert("ca-derivations"); + return res; +} Store::Store(const Params & params) : StoreConfig(params) @@ -392,11 +414,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) bool Store::isValidPath(const StorePath & storePath) { - std::string hashPart(storePath.hashPart()); - { auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); + auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; return res->didExist(); @@ -404,11 +424,11 @@ bool Store::isValidPath(const StorePath & storePath) } if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); + auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart())); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } @@ -418,7 +438,7 @@ bool Store::isValidPath(const StorePath & storePath) if (diskCache && !valid) // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0); return valid; } @@ -465,13 +485,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) void Store::queryPathInfo(const StorePath & storePath, Callback> callback) noexcept { - std::string hashPart; + auto hashPart = std::string(storePath.hashPart()); try { - hashPart = storePath.hashPart(); - { - auto res = state.lock()->pathInfoCache.get(hashPart); + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; if (!res->didExist()) @@ -486,7 +504,7 @@ void Store::queryPathInfo(const StorePath & storePath, stats.narInfoReadAverted++; { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) @@ -501,7 +519,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -511,14 +529,12 @@ void Store::queryPathInfo(const StorePath & storePath, { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -526,6 +542,74 @@ void Store::queryPathInfo(const StorePath & storePath, }}); } +void Store::queryRealisation(const DrvOutput & id, + Callback> callback) noexcept +{ + + try { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] + = diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + callback(maybeCachedRealisation); + return; + case NarInfoDiskCache::oInvalid: + debug( + "Returning a cached missing realisation for %s", + id.to_string()); + callback(nullptr); + return; + case NarInfoDiskCache::oUnknown: + break; + } + } + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr + = std::make_shared(std::move(callback)); + + queryRealisationUncached( + id, + { [this, id, callbackPtr]( + std::future> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + if (info) + diskCache->upsertRealisation(getUri(), *info); + else + diskCache->upsertAbsentRealisation(getUri(), id); + } + + (*callbackPtr)(std::shared_ptr(info)); + + } catch (...) { + callbackPtr->rethrow(); + } + } }); +} + +std::shared_ptr Store::queryRealisation(const DrvOutput & id) +{ + using RealPtr = std::shared_ptr; + std::promise promise; + + queryRealisation(id, + {[&](std::future result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} void Store::substitutePaths(const StorePathSet & paths) { @@ -627,6 +711,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths, } +StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths) +{ + StorePathSet paths; + + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath)); + + computeFSClosure({storePath}, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = paths; + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = derivationFromPath(j); + for (auto & k : drv.outputsAndOptPaths(*this)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + computeFSClosure(*k.second.second, paths); + } + } + } + + return paths; +} + + void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, @@ -727,30 +847,43 @@ const Store::Stats & Store::getStats() } -void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) +static std::string makeCopyPathMessage( + std::string_view srcUri, + std::string_view dstUri, + std::string_view storePath) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + return srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri); +} + +void copyStorePath( + Store & srcStore, + Store & dstStore, + const StorePath & storePath, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) - : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), - {srcStore->printStorePath(storePath), srcUri, dstUri}); + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); uint64_t total = 0; // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; } @@ -767,37 +900,61 @@ void copyStorePath(ref srcStore, ref dstStore, act.progress(total, info->narSize); }); TeeSink tee { sink, progressSink }; - srcStore->narFromPath(storePath, tee); + srcStore.narFromPath(storePath, tee); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri()); }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore.addToStore(*info, *source, repair, checkSigs); } -std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +std::map copyPaths( + Store & srcStore, + Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) { StorePathSet storePaths; - std::set realisations; + std::set toplevelRealisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { - settings.requireExperimentalFeature("ca-derivations"); - realisations.insert(*realisation); + settings.requireExperimentalFeature(Xp::CaDerivations); + toplevelRealisations.insert(*realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + + ThreadPool pool; + try { - for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation, checkSigs); - } + // Copy the realisation closure + processGraph( + pool, Realisation::closure(srcStore, toplevelRealisations), + [&](const Realisation & current) -> std::set { + std::set children; + for (const auto & [drvOutput, _] : current.dependentRealisations) { + auto currentChild = srcStore.queryRealisation(drvOutput); + if (!currentChild) + throw Error( + "incomplete realisation closure: '%s' is a " + "dependency of '%s' but isn't registered", + drvOutput.to_string(), current.id.to_string()); + children.insert(*currentChild); + } + return children; + }, + [&](const Realisation& current) -> void { + dstStore.registerDrvOutput(current, checkSigs); + }); } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. - if (e.missingFeature == "ca-derivations") + if (e.missingFeature == Xp::CaDerivations) ignoreException(); else throw; @@ -806,10 +963,15 @@ std::map copyPaths(ref srcStore, ref dstStor return pathsMap; } -std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +std::map copyPaths( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) { - auto valid = dstStore->queryValidPaths(storePaths, substitute); + auto valid = dstStore.queryValidPaths(storePaths, substitute); StorePathSet missing; for (auto & path : storePaths) @@ -819,9 +981,31 @@ std::map copyPaths(ref srcStore, ref dstStor for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + auto sorted = srcStore.topoSortPaths(missing); + std::reverse(sorted.begin(), sorted.end()); + + auto source = sinkToSource([&](Sink & sink) { + sink << sorted.size(); + for (auto & storePath : sorted) { + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore.queryPathInfo(storePath); + info->write(sink, srcStore, 16); + srcStore.narFromPath(storePath, sink); + } + }); + + dstStore.addMultipleToStore(*source, repair, checkSigs); + + #if 0 std::atomic nrDone{0}; std::atomic nrFailed{0}; std::atomic bytesExpected{0}; @@ -837,18 +1021,21 @@ std::map copyPaths(ref srcStore, ref dstStor StorePathSet(missing.begin(), missing.end()), [&](const StorePath & storePath) { - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePath), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); } pathsMap.insert_or_assign(storePath, storePathForDst); - if (dstStore->isValidPath(storePath)) { + if (dstStore.isValidPath(storePath)) { nrDone++; showProgress(); return StorePathSet(); @@ -863,19 +1050,22 @@ std::map copyPaths(ref srcStore, ref dstStor [&](const StorePath & storePath) { checkInterrupt(); - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePath), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); } pathsMap.insert_or_assign(storePath, storePathForDst); - if (!dstStore->isValidPath(storePathForDst)) { + if (!dstStore.isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); showProgress(); try { @@ -884,7 +1074,7 @@ std::map copyPaths(ref srcStore, ref dstStor nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); showProgress(); return; } @@ -893,9 +1083,27 @@ std::map copyPaths(ref srcStore, ref dstStor nrDone++; showProgress(); }); + #endif + return pathsMap; } +void copyClosure( + Store & srcStore, + Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + if (&srcStore == &dstStore) return; + + RealisedPath::Set closure; + RealisedPath::closure(srcStore, paths, closure); + + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) { std::string path; @@ -968,10 +1176,10 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const if (! ca) return false; auto caPath = std::visit(overloaded { - [&](TextHash th) { + [&](const TextHash & th) { return store.makeTextPath(path.name(), th.hash, references); }, - [&](FixedOutputHash fsh) { + [&](const FixedOutputHash & fsh) { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f66298991..aa44651d4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -180,6 +180,8 @@ struct StoreConfig : public Config StoreConfig() = delete; + StringSet getDefaultSystemFeatures(); + virtual ~StoreConfig() { } virtual const std::string name() = 0; @@ -196,7 +198,7 @@ struct StoreConfig : public Config Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; - Setting systemFeatures{this, settings.systemFeatures, + Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; @@ -230,7 +232,6 @@ protected: struct State { - // FIXME: fix key LRUCache pathInfoCache; }; @@ -368,6 +369,14 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /* Query the information about a realisation. */ + std::shared_ptr queryRealisation(const DrvOutput &); + + /* Asynchronous version of queryRealisation(). */ + void queryRealisation(const DrvOutput &, + Callback> callback) noexcept; + + /* Check whether the given valid path info is sufficiently attested, by either being signed by a trusted public key or content-addressed, in order to be included in the given store. @@ -392,11 +401,11 @@ protected: virtual void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept = 0; + virtual void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept = 0; public: - virtual std::optional queryRealisation(const DrvOutput &) = 0; - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -428,9 +437,10 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. If a path - does not have substitute info, it's omitted from the resulting - ‘infos’ map. */ + sizes) of a map of paths to their optional ca values. The info + of the first succeeding substituter for each path will be + returned. If a path does not have substitute info, it's omitted + from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; @@ -438,13 +448,19 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + /* Import multiple paths into the store. */ + virtual void addMultipleToStore( + Source & source, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -460,7 +476,8 @@ public: `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is @@ -495,7 +512,8 @@ public: not derivations, substitute them. */ virtual void buildPaths( const std::vector & paths, - BuildMode buildMode = bmNormal); + BuildMode buildMode = bmNormal, + std::shared_ptr evalStore = nullptr); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). @@ -541,7 +559,7 @@ public: /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ virtual void addTempRoot(const StorePath & path) - { warn("not creating temp root, store doesn't support GC"); } + { debug("not creating temporary root, store doesn't support GC"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed @@ -551,26 +569,6 @@ public: virtual void addIndirectRoot(const Path & path) { unsupported("addIndirectRoot"); } - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees ours. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() { }; - /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink outside of the Nix store that point to `storePath'. If @@ -695,6 +693,11 @@ public: const Stats & getStats(); + /* Computes the full closure of of a set of store-paths for e.g. + derivations that need this information for `exportReferencesGraph`. + */ + StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr getBuildLog(const StorePath & path) @@ -730,6 +733,11 @@ public: virtual void createUser(const std::string & userName, uid_t userId) { } + /* + * Synchronises the options of the client with those of the daemon + * (a no-op when there’s no daemon) + */ + virtual void setOptions() { } protected: Stats stats; @@ -744,8 +752,12 @@ protected: /* Copy a path from one store to another. */ -void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); +void copyStorePath( + Store & srcStore, + Store & dstStore, + const StorePath & storePath, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied @@ -754,17 +766,27 @@ void copyStorePath(ref srcStore, ref dstStore, of store paths is not automatically closed; use copyClosure() for that. Returns a map of what each path was copied to the dstStore as. */ -std::map copyPaths(ref srcStore, ref dstStore, +std::map copyPaths( + Store & srcStore, Store & dstStore, const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -std::map copyPaths(ref srcStore, ref dstStore, - const StorePathSet& paths, + +std::map copyPaths( + Store & srcStore, Store & dstStore, + const StorePathSet & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); +/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +void copyClosure( + Store & srcStore, Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Remove the temporary roots file for this process. Any temporary root becomes garbage after this point unless it has been registered @@ -864,4 +886,9 @@ std::pair splitUriAndParams(const std::string & uri) std::optional getDerivationCA(const BasicDerivation & drv); +std::map drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath); + } diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk new file mode 100644 index 000000000..f74295d97 --- /dev/null +++ b/src/libstore/tests/local.mk @@ -0,0 +1,15 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_DIR := $(d) + +libstore-tests_INSTALL_DIR := + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil + +libstore-tests_LIBS = libstore libutil + +libstore-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc new file mode 100644 index 000000000..f51052b14 --- /dev/null +++ b/src/libstore/tests/machines.cc @@ -0,0 +1,169 @@ +#include "machines.hh" +#include "globals.hh" + +#include + +using testing::Contains; +using testing::ElementsAre; +using testing::EndsWith; +using testing::Eq; +using testing::Field; +using testing::SizeIs; + +using nix::absPath; +using nix::FormatError; +using nix::getMachines; +using nix::Machine; +using nix::Machines; +using nix::pathExists; +using nix::Settings; +using nix::settings; + +class Environment : public ::testing::Environment { + public: + void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } +}; + +testing::Environment* const foo_env = + testing::AddGlobalTestEnvironment(new Environment); + +TEST(machines, getMachinesWithEmptyBuilders) { + settings.builders = ""; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesUriOnly) { + settings.builders = "nix@scratchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesDefaults) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesWithNewLineSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithSemicolonSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + EXPECT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { + settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, + getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { + settings.builders = + "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" + "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" + "KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, getMachinesWithMultiOptions) { + settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); +} + +TEST(machines, getMachinesWithIncorrectFormat) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; + EXPECT_THROW(getMachines(), FormatError); +} + +TEST(machines, getMachinesWithCorrectFileReference) { + auto path = absPath("src/libstore/tests/test-data/machines.valid"); + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { + auto path = "/dev/null"; + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithIncorrectFileReference) { + settings.builders = std::string("@") + absPath("/not/a/file"); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { + settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + EXPECT_THROW(getMachines(), FormatError); +} diff --git a/src/libstore/tests/references.cc b/src/libstore/tests/references.cc new file mode 100644 index 000000000..d91d1cedd --- /dev/null +++ b/src/libstore/tests/references.cc @@ -0,0 +1,45 @@ +#include "references.hh" + +#include + +namespace nix { + +TEST(references, scan) +{ + std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0"; + std::string hash2 = "zc842j0rz61mjsp3h3wp5ly71ak6qgdn"; + + { + RefScanSink scanner(StringSet{hash1}); + auto s = "foobar"; + scanner(s); + ASSERT_EQ(scanner.getResult(), StringSet{}); + } + + { + RefScanSink scanner(StringSet{hash1}); + auto s = "foobar" + hash1 + "xyzzy"; + scanner(s); + ASSERT_EQ(scanner.getResult(), StringSet{hash1}); + } + + { + RefScanSink scanner(StringSet{hash1, hash2}); + auto s = "foobar" + hash1 + "xyzzy" + hash2; + scanner(((std::string_view) s).substr(0, 10)); + scanner(((std::string_view) s).substr(10, 5)); + scanner(((std::string_view) s).substr(15, 5)); + scanner(((std::string_view) s).substr(20)); + ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2})); + } + + { + RefScanSink scanner(StringSet{hash1, hash2}); + auto s = "foobar" + hash1 + "xyzzy" + hash2; + for (auto & i : s) + scanner(std::string(1, i)); + ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2})); + } +} + +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/src/libstore/tests/test-data/machines.bad_format new file mode 100644 index 000000000..7255a1216 --- /dev/null +++ b/src/libstore/tests/test-data/machines.bad_format @@ -0,0 +1 @@ +nix@scratchy.labs.cs.uu.nl - - eight diff --git a/src/libstore/tests/test-data/machines.valid b/src/libstore/tests/test-data/machines.valid new file mode 100644 index 000000000..1a6c8017c --- /dev/null +++ b/src/libstore/tests/test-data/machines.valid @@ -0,0 +1,3 @@ +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDWWV5R1laNTNzd1VjMUZNSHBWL1BCcXlKaFR5S1JoRkpWWVRpRHlQN2h5c1JGa0w4VDlLOGdhL2Y2L3c3QjN2SjNHSFRIUFkybENiUEdZbGNLd2h6M2ZRbFNNOEViNi95b3ZLajdvM1FsMEx5Y0dzdGJvRmcwWkZKNldncUxsR0ltS0NobUlxOGZ3TW5ZTWUxbnRQeTBUZFZjSU1tOTV3YzF3SjBMd2c3cEVMRmtHazdkeTVvYnM4a3lGZ0pORDVRSmFwQWJjeWp4Z1QzdzdMcktNZ2xzeWhhd01JNVpkMGZsQTVudW5OZ3pid3plYVhLaUsyTW0vdGJXYTU1YTd4QmNYdHpIZGlPSWdSajJlRWxaMGh5bk10YjBmcklsdmxIcEtLaVFaZ3pQdCtIVXQ2bXpRMkRVME52MGYyYnNSU0krOGpJU2pQcmdlcVVHRldMUzVIUTg2N2xSMlpiaWtyclhZNTdqbVFEZk5DRHY1VFBHZU9UekFEd2pjMDc2aFZ3VFJCd3VTZFhtaWNxTS95b3lrWitkV1dnZ25MenE5QU1tdlNZcDhmZkZDcS9CSDBZNUFXWTFHay9vS3hMVTNaOWt3ZDd2UWNFQWFCQ2dxdnVZRGdTaHE1RlhndDM3OVZESWtEL05ZSTg2QXVvajVDRmVNTzlRM2pJSlRadlh6c1VldjVoSnA2djcxSVh5ODVtbTY5R20zcXdicVE1SjVQZDU1Um56SitpaW5BNjZxTEFSc0Y4amNsSnd5ekFXclBoYU9DRVY2bjVMeVhVazhzMW9EVVR4V1pWN25rVkFTbHJ0MllGcjN5dzdjRTRXQVhsemhHcDhocmdLMVVkMUlyeDVnZWRaSnBWcy9uNWVybmJFMUxmb2x5UHUvRUFIWlh6VGd4dHVDUFNobXc9PQo= diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index cac4fa036..5c38323cd 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -45,30 +45,20 @@ std::string UDSRemoteStore::getUri() } +void UDSRemoteStore::Connection::closeWrite() +{ + shutdown(fd.get(), SHUT_WR); +} + + ref UDSRemoteStore::openConnection() { auto conn = make_ref(); /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); + conn->fd = createUnixDomainSocket(); - string socketPath = path ? *path : settings.nixDaemonSocketFile; - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", socketPath); - strcpy(addr.sun_path, socketPath.c_str()); - - if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at '%1%'", socketPath); + nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index ddc7716cd..f8dfcca70 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -40,6 +40,12 @@ public: private: + struct Connection : RemoteStore::Connection + { + AutoCloseFD fd; + void closeWrite() override; + }; + ref openConnection() override; std::optional path; }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 001ed25e3..93cf546d2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 29) +#define PROTOCOL_VERSION (1 << 8 | 32) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -55,6 +55,7 @@ typedef enum { wopQueryDerivationOutputMap = 41, wopRegisterDrvOutput = 42, wopQueryRealisation = 43, + wopAddMultipleToStore = 44, } WorkerOp; diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index ae741f867..38305e71c 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -9,7 +9,7 @@ namespace nix { #define ANSI_ITALIC "\e[3m" #define ANSI_RED "\e[31;1m" #define ANSI_GREEN "\e[32;1m" -#define ANSI_YELLOW "\e[33;1m" +#define ANSI_WARNING "\e[35;1m" #define ANSI_BLUE "\e[34;1m" #define ANSI_MAGENTA "\e[35;1m" #define ANSI_CYAN "\e[36;1m" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index ed0eb2fb5..d78ec2b93 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~"; PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, size_t size, +static void dumpContents(const Path & path, off_t size, Sink & sink) { sink << "contents" << size; @@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) sink << "type" << "regular"; if (st.st_mode & S_IXUSR) sink << "executable" << ""; - dumpContents(path, (size_t) st.st_size, sink); + dumpContents(path, st.st_size, sink); } else if (S_ISDIR(st.st_mode)) { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index afed0670f..9df279faf 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -331,6 +331,7 @@ MultiCommand::MultiCommand(const Commands & commands_) if (i == commands.end()) throw UsageError("'%s' is not a recognised command", s); command = {s, i->second()}; + command->second->parent = this; }} }); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c08ba8abd..7521b3065 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -12,6 +12,8 @@ namespace nix { enum HashType : char; +class MultiCommand; + class Args { public: @@ -89,6 +91,14 @@ protected: }) , arity(1) { } + + template + Handler(std::optional * dest) + : fun([=](std::vector ss) { + *dest = string2IntWithUnitPrefix(ss[0]); + }) + , arity(1) + { } }; /* Options. */ @@ -169,11 +179,13 @@ public: virtual nlohmann::json toJSON(); friend class MultiCommand; + + MultiCommand * parent = nullptr; }; /* A command is an argument parser that can be executed by calling its run() method. */ -struct Command : virtual Args +struct Command : virtual public Args { friend class MultiCommand; @@ -193,7 +205,7 @@ typedef std::map()>> Commands; /* An argument parser that supports multiple subcommands, i.e. ‘ ’. */ -class MultiCommand : virtual Args +class MultiCommand : virtual public Args { public: Commands commands; diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh new file mode 100644 index 000000000..779b9b2d5 --- /dev/null +++ b/src/libutil/closure.hh @@ -0,0 +1,69 @@ +#include +#include +#include "sync.hh" + +using std::set; + +namespace nix { + +template +using GetEdgesAsync = std::function> &)>)>; + +template +void computeClosure( + const set startElts, + set & res, + GetEdgesAsync getEdgesAsync +) +{ + struct State + { + size_t pending; + set & res; + std::exception_ptr exc; + }; + + Sync state_(State{0, res, 0}); + + std::function enqueue; + + std::condition_variable done; + + enqueue = [&](const T & current) -> void { + { + auto state(state_.lock()); + if (state->exc) return; + if (!state->res.insert(current).second) return; + state->pending++; + } + + getEdgesAsync(current, [&](std::promise> & prom) { + try { + auto children = prom.get_future().get(); + for (auto & child : children) + enqueue(child); + { + auto state(state_.lock()); + assert(state->pending); + if (!--state->pending) done.notify_one(); + } + } catch (...) { + auto state(state_.lock()); + if (!state->exc) state->exc = std::current_exception(); + assert(state->pending); + if (!--state->pending) done.notify_one(); + }; + }); + }; + + for (auto & startElt : startElts) + enqueue(startElt); + + { + auto state(state_.lock()); + while (state->pending) state.wait(done); + if (state->exc) std::rethrow_exception(state->exc); + } +} + +} diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 0315dc506..eecd5b819 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -25,6 +25,8 @@ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) + GENERATE_LEQ(args) \ + GENERATE_NEQ(args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 7e725cae1..f80ca664c 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -12,12 +12,12 @@ #include #include -#include - #include namespace nix { +static const int COMPRESSION_LEVEL_DEFAULT = -1; + // Don't feed brotli too much at once. struct ChunkedCompressionSink : CompressionSink { @@ -67,14 +67,16 @@ struct ArchiveCompressionSink : CompressionSink Sink & nextSink; struct archive * archive; - ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + { archive = archive_write_new(); if (!archive) throw Error("failed to initialize libarchive"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); check(archive_write_set_format_raw(archive)); - if (format == "xz" && parallel) { + if (parallel) check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); - } + if (level != COMPRESSION_LEVEL_DEFAULT) + check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str())); // disable internal buffering check(archive_write_set_bytes_per_block(archive, 0)); // disable output padding @@ -128,7 +130,11 @@ private: struct NoneSink : CompressionSink { Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } + NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + { + if (level != COMPRESSION_LEVEL_DEFAULT) + warn("requested compression level '%d' not supported by compression method 'none'", level); + } void finish() override { flush(); } void write(std::string_view data) override { nextSink(data); } }; @@ -259,13 +265,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink } }; -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { std::vector la_supports = { "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" }; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { - return make_ref(nextSink, method, parallel); + return make_ref(nextSink, method, parallel, level); } if (method == "none") return make_ref(nextSink); @@ -275,10 +281,10 @@ ref makeCompressionSink(const std::string & method, Sink & next throw UnknownCompressionMethod("unknown compression method '%s'", method); } -ref compress(const std::string & method, const std::string & in, const bool parallel) +ref compress(const std::string & method, const std::string & in, const bool parallel, int level) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink, parallel); + auto sink = makeCompressionSink(method, ssink, parallel, level); (*sink)(in); sink->finish(); return ssink.s; diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 338a0d9f2..9b1e4a9d4 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -19,9 +19,9 @@ ref decompress(const std::string & method, const std::string & in); std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink); -ref compress(const std::string & method, const std::string & in, const bool parallel = false); +ref compress(const std::string & method, const std::string & in, const bool parallel = false, int level = -1); -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index bda07cd55..92ab265d3 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "experimental-features.hh" #include @@ -152,6 +153,16 @@ nlohmann::json Config::toJSON() return res; } +std::string Config::toKeyValue() +{ + auto res = std::string(); + for (auto & s : _settings) + if (!s.second.isAlias) { + res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); + } + return res; +} + void Config::convertToArgs(Args & args, const std::string & category) { for (auto & s : _settings) @@ -167,11 +178,6 @@ AbstractSetting::AbstractSetting( { } -void AbstractSetting::setDefault(const std::string & str) -{ - if (!overridden) set(str); -} - nlohmann::json AbstractSetting::toJSON() { return nlohmann::json(toJSONObject()); @@ -308,6 +314,31 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } +template<> void BaseSetting>::set(const std::string & str, bool append) +{ + if (!append) value.clear(); + for (auto & s : tokenizeString(str)) { + auto thisXpFeature = parseExperimentalFeature(s); + if (thisXpFeature) + value.insert(thisXpFeature.value()); + else + warn("unknown experimental feature '%s'", s); + } +} + +template<> bool BaseSetting>::isAppendable() +{ + return true; +} + +template<> std::string BaseSetting>::to_string() const +{ + StringSet stringifiedXpFeatures; + for (auto & feature : value) + stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); + return concatStringsSep(" ", stringifiedXpFeatures); +} + template<> void BaseSetting::set(const std::string & str, bool append) { if (!append) value.clear(); @@ -343,6 +374,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; +template class BaseSetting>; void PathSetting::set(const std::string & str, bool append) { @@ -385,6 +417,16 @@ nlohmann::json GlobalConfig::toJSON() return res; } +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + void GlobalConfig::convertToArgs(Args & args, const std::string & category) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index bf81b4892..736810bf3 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -99,6 +99,12 @@ public: */ virtual nlohmann::json toJSON() = 0; + /** + * Outputs all settings in a key-value pair format suitable to be used as + * `nix.conf` + */ + virtual std::string toKeyValue() = 0; + /** * Converts settings to `Args` to be used on the command line interface * - args: args to write to @@ -169,6 +175,8 @@ public: nlohmann::json toJSON() override; + std::string toKeyValue() override; + void convertToArgs(Args & args, const std::string & category) override; }; @@ -186,8 +194,6 @@ public: bool overridden = false; - void setDefault(const std::string & str); - protected: AbstractSetting( @@ -245,6 +251,7 @@ public: bool operator !=(const T & v2) const { return value != v2; } void operator =(const T & v) { assign(v); } virtual void assign(const T & v) { value = v; } + void setDefault(const T & v) { if (!overridden) value = v; } void set(const std::string & str, bool append = false) override; @@ -330,6 +337,8 @@ struct GlobalConfig : public AbstractConfig nlohmann::json toJSON() override; + std::string toKeyValue() override; + void convertToArgs(Args & args, const std::string & category) override; struct Register diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 0eea3455d..203d79087 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -185,15 +185,15 @@ void printAtPos(const ErrPos & pos, std::ostream & out) if (pos) { switch (pos.origin) { case foFile: { - out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); + out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); break; } case foString: { - out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } case foStdin: { - out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } default: @@ -232,7 +232,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } case Verbosity::lvlWarn: { - prefix = ANSI_YELLOW "warning"; + prefix = ANSI_WARNING "warning"; break; } case Verbosity::lvlInfo: { @@ -252,7 +252,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } case Verbosity::lvlDebug: { - prefix = ANSI_YELLOW "debug"; + prefix = ANSI_WARNING "debug"; break; } default: diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc new file mode 100644 index 000000000..b49f47e1d --- /dev/null +++ b/src/libutil/experimental-features.cc @@ -0,0 +1,59 @@ +#include "experimental-features.hh" +#include "util.hh" + +#include "nlohmann/json.hpp" + +namespace nix { + +std::map stringifiedXpFeatures = { + { Xp::CaDerivations, "ca-derivations" }, + { Xp::Flakes, "flakes" }, + { Xp::NixCommand, "nix-command" }, + { Xp::RecursiveNix, "recursive-nix" }, + { Xp::NoUrlLiterals, "no-url-literals" }, +}; + +const std::optional parseExperimentalFeature(const std::string_view & name) +{ + using ReverseXpMap = std::map; + + static auto reverseXpMap = []() + { + auto reverseXpMap = std::make_unique(); + for (auto & [feature, name] : stringifiedXpFeatures) + (*reverseXpMap)[name] = feature; + return reverseXpMap; + }(); + + if (auto feature = get(*reverseXpMap, name)) + return *feature; + else + return std::nullopt; +} + +std::string_view showExperimentalFeature(const ExperimentalFeature feature) +{ + return stringifiedXpFeatures.at(feature); +} + +std::set parseFeatures(const std::set & rawFeatures) +{ + std::set res; + for (auto & rawFeature : rawFeatures) { + if (auto feature = parseExperimentalFeature(rawFeature)) + res.insert(*feature); + } + return res; +} + +MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) + : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + , missingFeature(feature) +{} + +std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & feature) +{ + return str << showExperimentalFeature(feature); +} + +} diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh new file mode 100644 index 000000000..291a58e32 --- /dev/null +++ b/src/libutil/experimental-features.hh @@ -0,0 +1,56 @@ +#pragma once + +#include "comparator.hh" +#include "error.hh" +#include "nlohmann/json_fwd.hpp" +#include "types.hh" + +namespace nix { + +/** + * The list of available experimental features. + * + * If you update this, don’t forget to also change the map defining their + * string representation in the corresponding `.cc` file. + **/ +enum struct ExperimentalFeature +{ + CaDerivations, + Flakes, + NixCommand, + RecursiveNix, + NoUrlLiterals +}; + +/** + * Just because writing `ExperimentalFeature::CaDerivations` is way too long + */ +using Xp = ExperimentalFeature; + +const std::optional parseExperimentalFeature( + const std::string_view & name); +std::string_view showExperimentalFeature(const ExperimentalFeature); + +std::ostream & operator<<( + std::ostream & str, + const ExperimentalFeature & feature); + +/** + * Parse a set of strings to the corresponding set of experimental features, + * ignoring (but warning for) any unkwown feature. + */ +std::set parseFeatures(const std::set &); + +class MissingExperimentalFeature : public Error +{ +public: + ExperimentalFeature missingFeature; + + MissingExperimentalFeature(ExperimentalFeature); + virtual const char * sname() const override + { + return "MissingExperimentalFeature"; + } +}; + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 85c0e9429..fd335b811 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -82,7 +82,7 @@ struct yellowtxt template std::ostream & operator<<(std::ostream & out, const yellowtxt & y) { - return out << ANSI_YELLOW << y.value << ANSI_NORMAL; + return out << ANSI_WARNING << y.value << ANSI_NORMAL; } template diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 3a6415ee3..f880c0fc5 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,7 +6,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d2e801175..f8a121ed1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -27,7 +27,7 @@ Logger * logger = makeSimpleLogger(true); void Logger::warn(const std::string & msg) { - log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_WARNING "warning:" ANSI_NORMAL " " + msg); } void Logger::writeToStdout(std::string_view s) @@ -46,7 +46,7 @@ public: : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; - tty = isatty(STDERR_FILENO); + tty = shouldANSI(); } bool isVerbose() override { @@ -163,7 +163,7 @@ struct JSONLogger : Logger { void write(const nlohmann::json & json) { - prevLogger.log(lvlError, "@nix " + json.dump()); + prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)); } void log(Verbosity lvl, const FormatOrString & fs) override diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 0be2a7e74..347b81f73 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -17,7 +17,7 @@ private: public: - ref(const ref & r) + ref(const ref & r) : p(r.p) { } @@ -73,6 +73,16 @@ public: return ref((std::shared_ptr) p); } + bool operator == (const ref & other) const + { + return p == other.p; + } + + bool operator != (const ref & other) const + { + return p != other.p; + } + private: template @@ -89,4 +99,47 @@ make_ref(Args&&... args) return ref(p); } + +/* A non-nullable pointer. + This is similar to a C++ "& reference", but mutable. + This is similar to ref but backed by a regular pointer instead of a smart pointer. + */ +template +class ptr { +private: + T * p; + +public: + ptr(const ptr & r) + : p(r.p) + { } + + explicit ptr(T * p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ptr"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + bool operator == (const ptr & other) const + { + return p == other.p; + } + + bool operator != (const ptr & other) const + { + return p != other.p; + } +}; + } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 374b48d79..16f3476c2 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -244,7 +244,8 @@ std::unique_ptr sourceToSink(std::function fun) if (!cur.empty()) (*coro)(false); } - void finish() { + void finish() override + { if (!coro) return; if (!*coro) abort(); (*coro)(true); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 24905130d..50e691a3d 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -39,32 +39,30 @@ void TarArchive::check(int err, const std::string & reason) throw Error(reason, archive_error_string(this->archive)); } -TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) +TarArchive::TarArchive(Source & source, bool raw) + : source(&source), buffer(4096) { - this->archive = archive_read_new(); - this->source = &source; - - if (!raw) { - archive_read_support_filter_all(archive); + init(); + if (!raw) archive_read_support_format_all(archive); - } else { - archive_read_support_filter_all(archive); + else archive_read_support_format_raw(archive); - archive_read_support_format_empty(archive); - } check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } - TarArchive::TarArchive(const Path & path) { - this->archive = archive_read_new(); - - archive_read_support_filter_all(archive); + init(); archive_read_support_format_all(archive); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } +void TarArchive::init() +{ + archive = archive_read_new(); + archive_read_support_filter_all(archive); +} + void TarArchive::close() { check(archive_read_close(this->archive), "Failed to close archive (%s)"); @@ -87,13 +85,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) struct archive_entry * entry; int r = archive_read_next_header(archive.archive, &entry); if (r == ARCHIVE_EOF) break; - else if (r == ARCHIVE_WARN) + auto name = archive_entry_pathname(entry); + if (!name) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) warn(archive_error_string(archive.archive)); else archive.check(r); archive_entry_set_pathname(entry, - (destDir + "/" + archive_entry_pathname(entry)).c_str()); + (destDir + "/" + name).c_str()); archive.check(archive_read_extract(archive.archive, entry, flags)); } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 4d9141fd4..f107a7e2e 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -17,10 +17,13 @@ struct TarArchive { // disable copy constructor TarArchive(const TarArchive &) = delete; + void init(); + void close(); ~TarArchive(); }; + void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); diff --git a/src/libutil/tests/closure.cc b/src/libutil/tests/closure.cc new file mode 100644 index 000000000..7597e7807 --- /dev/null +++ b/src/libutil/tests/closure.cc @@ -0,0 +1,70 @@ +#include "closure.hh" +#include + +namespace nix { + +using namespace std; + +map> testGraph = { + { "A", { "B", "C", "G" } }, + { "B", { "A" } }, // Loops back to A + { "C", { "F" } }, // Indirect reference + { "D", { "A" } }, // Not reachable, but has backreferences + { "E", {} }, // Just not reachable + { "F", {} }, + { "G", { "G" } }, // Self reference +}; + +TEST(closure, correctClosure) { + set aClosure; + set expectedClosure = {"A", "B", "C", "F", "G"}; + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promisedNodes; + promisedNodes.set_value(testGraph[currentNode]); + processEdges(promisedNodes); + } + ); + + ASSERT_EQ(aClosure, expectedClosure); +} + +TEST(closure, properlyHandlesDirectExceptions) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + throw TestExn(); + } + ), + TestExn + ); +} + +TEST(closure, properlyHandlesExceptionsInPromise) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promise; + try { + throw TestExn(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + processEdges(promise); + } + ), + TestExn + ); +} + +} diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index d990e5499..cef3bd481 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -336,7 +336,7 @@ namespace nix { ASSERT_STREQ( hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(), - "only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " "); + "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " "); } @@ -344,7 +344,7 @@ namespace nix { ASSERT_STREQ( hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), - "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); + "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 58df9c5ac..92972ed14 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -4,6 +4,8 @@ #include #include +#include + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ @@ -282,6 +284,17 @@ namespace nix { ASSERT_EQ(decoded, s); } + TEST(base64Encode, encodeAndDecodeNonPrintable) { + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); + } + /* ---------------------------------------------------------------------------- * base64Decode * --------------------------------------------------------------------------*/ @@ -294,6 +307,10 @@ namespace nix { ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } + TEST(base64Decode, decodeThrowsOnInvalidChar) { + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); + } + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/libutil/url.cc b/src/libutil/url.cc index c1bab866c..f6232d255 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url) auto isFile = scheme.find("file") != std::string::npos; if (authority && *authority != "" && isFile) - throw Error("file:// URL '%s' has unexpected authority '%s'", + throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); if (isFile && path.empty()) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5f597bf06..defb77a10 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -4,16 +4,18 @@ #include "finally.hh" #include "serialise.hh" +#include #include #include +#include #include #include #include -#include +#include #include +#include #include #include -#include #include #include @@ -155,6 +157,9 @@ Path canonPath(const Path & path, bool resolveSymlinks) s.clear(); /* restart for symlinks pointing to absolute path */ } else { s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } } } } @@ -410,7 +415,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) } int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (!fd) + if (fd == -1) throw SysError("opening directory '%1%'", path); AutoCloseDir dir(fdopendir(fd)); if (!dir) @@ -432,12 +437,9 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed) if (dir == "") dir = "/"; - AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; if (!dirfd) { - // This really shouldn't fail silently, but it's left this way - // for backwards compatibility. if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); } @@ -560,7 +562,7 @@ Path getConfigDir() std::vector getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector result = tokenizeString>(configDirs, ":"); result.insert(result.begin(), configHome); return result; @@ -901,7 +903,7 @@ int Pid::wait() return status; } if (errno != EINTR) - throw SysError("cannot get child exit status"); + throw SysError("cannot get exit status of PID %d", pid); checkInterrupt(); } } @@ -937,9 +939,6 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - ProcessOptions options; - options.allowVfork = false; - Pid pid = startProcess([&]() { if (setuid(uid) == -1) @@ -962,7 +961,7 @@ void killUser(uid_t uid) } _exit(0); - }, options); + }); int status = pid.wait(); if (status != 0) @@ -1032,17 +1031,10 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional & input) { - RunOptions opts(program, args); - opts.searchPath = searchPath; - // This allows you to refer to a program with a pathname relative to the - // PATH variable. - opts.input = input; - - auto res = runProgram(opts); + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); if (!statusOk(res.first)) throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); @@ -1051,9 +1043,8 @@ string runProgram(Path program, bool searchPath, const Strings & args, } // Output = error code + "standard out" output stream -std::pair runProgram(const RunOptions & options_) +std::pair runProgram(RunOptions && options) { - RunOptions options(options_); StringSink sink; options.standardOut = &sink; @@ -1091,8 +1082,7 @@ void runProgram2(const RunOptions & options) // vfork implies that the environment of the main process and the fork will // be shared (technically this is undefined, but in practice that's the // case), so we can't use it if we alter the environment - if (options.environment) - processOptions.allowVfork = false; + processOptions.allowVfork = !options.environment; /* Fork. */ Pid pid = startProcess([&]() { @@ -1215,7 +1205,7 @@ void closeOnExec(int fd) ////////////////////////////////////////////////////////////////////// -bool _isInterrupted = false; +std::atomic _isInterrupted = false; static thread_local bool interruptThrown = false; thread_local std::function interruptCheck; @@ -1369,6 +1359,12 @@ void ignoreException() } } +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !getEnv("NO_COLOR").has_value(); +} std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) { @@ -1440,8 +1436,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } -static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; string base64Encode(std::string_view s) { @@ -1466,15 +1461,15 @@ string base64Encode(std::string_view s) string base64Decode(std::string_view s) { - bool init = false; - char decode[256]; - if (!init) { - // FIXME: not thread-safe. - memset(decode, -1, sizeof(decode)); + constexpr char npos = -1; + constexpr std::array base64DecodeChars = [&]() { + std::array result{}; + for (auto& c : result) + c = npos; for (int i = 0; i < 64; i++) - decode[(int) base64Chars[i]] = i; - init = true; - } + result[base64Chars[i]] = i; + return result; + }(); string res; unsigned int d = 0, bits = 0; @@ -1483,8 +1478,8 @@ string base64Decode(std::string_view s) if (c == '=') break; if (c == '\n') continue; - char digit = decode[(unsigned char) c]; - if (digit == -1) + char digit = base64DecodeChars[(unsigned char) c]; + if (digit == npos) throw Error("invalid character in Base64 string: '%c'", c); bits += 6; @@ -1637,9 +1632,39 @@ void setStackSize(size_t stackSize) #endif } -void restoreProcessContext() +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) + throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} + +void restoreProcessContext(bool restoreMounts) { restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } restoreAffinity(); @@ -1677,7 +1702,7 @@ std::unique_ptr createInterruptCallback(std::function } -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM #ifdef SOCK_CLOEXEC @@ -1686,19 +1711,16 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) , 0); if (!fdSocket) throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); + return fdSocket; +} - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (path.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", path); - strcpy(addr.sun_path, path.c_str()); - unlink(path.c_str()); +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); - if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%1%'", path); + bind(fdSocket.get(), path); if (chmod(path.c_str(), mode) == -1) throw SysError("changing permissions on '%1%'", path); @@ -1710,6 +1732,66 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) } +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + auto dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + auto dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + + string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); @@ -1719,8 +1801,10 @@ string showBytes(uint64_t bytes) // FIXME: move to libstore/build void commonChildInit(Pipe & logPipe) { + logger = makeSimpleLogger(); + const static string pathNullDevice = "/dev/null"; - restoreProcessContext(); + restoreProcessContext(false); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f84d0fb31..0bdb37a79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -259,10 +259,10 @@ void killUser(uid_t uid); pid to the caller. */ struct ProcessOptions { - string errorPrefix = "error: "; + string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; - bool allowVfork = true; + bool allowVfork = false; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); @@ -276,26 +276,20 @@ string runProgram(Path program, bool searchPath = false, struct RunOptions { + Path program; + bool searchPath = true; + Strings args; std::optional uid; std::optional gid; std::optional chdir; std::optional> environment; - Path program; - bool searchPath = true; - Strings args; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; - bool _killStderr = false; - - RunOptions(const Path & program, const Strings & args) - : program(program), args(args) { }; - - RunOptions & killStderr(bool v) { _killStderr = true; return *this; } }; -std::pair runProgram(const RunOptions & options); +std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); @@ -306,7 +300,15 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal masks, stack size, CPU affinity). */ -void restoreProcessContext(); +void restoreProcessContext(bool restoreMounts = true); + +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); class ExecError : public Error @@ -335,7 +337,7 @@ void closeOnExec(int fd); /* User interruption. */ -extern bool _isInterrupted; +extern std::atomic _isInterrupted; extern thread_local std::function interruptCheck; @@ -482,6 +484,9 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; +/* Determine whether ANSI escape sequences are appropriate for the + present output. */ +bool shouldANSI(); /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, @@ -514,6 +519,29 @@ std::optional get(const T & map, const typename T::key_ } +/* Remove and return the first item from a container. */ +template +std::optional remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/* Remove and return the first item from a container. */ +template +std::optional pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + template class Callback; @@ -574,9 +602,18 @@ extern PathFilter defaultPathFilter; /* Common initialisation performed in child processes. */ void commonChildInit(Pipe & logPipe); +/* Create a Unix domain socket. */ +AutoCloseFD createUnixDomainSocket(); + /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +/* Bind a Unix domain socket to a path. */ +void bind(int fd, const std::string & path); + +/* Connect to a Unix domain socket. */ +void connect(int fd, const std::string & path); + // A Rust/Python-like enumerate() iterator adapter. // Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9acbedda2..75576ef8a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -1,10 +1,15 @@ #include #include #include +#include #include #include #include +#include +#include + +#include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -100,7 +105,8 @@ static void main_nix_build(int argc, char * * argv) // List of environment variables kept for --pure std::set keepVars{ - "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", + "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", + "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; @@ -245,8 +251,9 @@ static void main_nix_build(int argc, char * * argv) throw UsageError("'-p' and '-E' are mutually exclusive"); auto store = openStore(); + auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.searchPath, store); + auto state = std::make_unique(myArgs.searchPath, evalStore, store); state->repair = repair; auto autoArgs = myArgs.getAutoArgs(*state); @@ -263,7 +270,7 @@ static void main_nix_build(int argc, char * * argv) if (packages) { std::ostringstream joined; - joined << "with import { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; + joined << "{...}@args: with import args; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; for (const auto & i : left) joined << '(' << i << ") "; joined << "]; } \"\""; @@ -296,8 +303,8 @@ static void main_nix_build(int argc, char * * argv) absolute = canonPath(absPath(i), true); } catch (Error & e) {}; auto [path, outputNames] = parsePathWithOutputs(absolute); - if (store->isStorePath(path) && hasSuffix(path, ".drv")) - drvs.push_back(DrvInfo(*state, store, absolute)); + if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) + drvs.push_back(DrvInfo(*state, evalStore, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ @@ -335,7 +342,7 @@ static void main_nix_build(int argc, char * * argv) printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (!dryRun) - store->buildPaths(paths, buildMode); + store->buildPaths(paths, buildMode, evalStore); }; if (runEnv) { @@ -343,9 +350,10 @@ static void main_nix_build(int argc, char * * argv) throw UsageError("nix-shell requires a single derivation"); auto & drvInfo = drvs.front(); - auto drv = store->derivationFromPath(store->parseStorePath(drvInfo.queryDrvPath())); + auto drv = evalStore->derivationFromPath(evalStore->parseStorePath(drvInfo.queryDrvPath())); std::vector pathsToBuild; + RealisedPath::Set pathsToCopy; /* Figure out what bash shell to use. If $NIX_BUILD_SHELL is not set, then build bashInteractive from @@ -364,7 +372,9 @@ static void main_nix_build(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); - pathsToBuild.push_back({store->parseStorePath(drv->queryDrvPath())}); + auto bashDrv = store->parseStorePath(drv->queryDrvPath()); + pathsToBuild.push_back({bashDrv}); + pathsToCopy.insert(bashDrv); shell = drv->queryOutPath() + "/bin/bash"; @@ -379,14 +389,25 @@ static void main_nix_build(int argc, char * * argv) for (const auto & input : drv.inputDrvs) if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) + { pathsToBuild.push_back({input.first, input.second}); - for (const auto & src : drv.inputSrcs) + pathsToCopy.insert(input.first); + } + for (const auto & src : drv.inputSrcs) { pathsToBuild.push_back({src}); + pathsToCopy.insert(src); + } buildPaths(pathsToBuild); if (dryRun) return; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto resolvedDrv = drv.tryResolve(*store); + assert(resolvedDrv && "Successfully resolved the derivation"); + drv = *resolvedDrv; + } + // Set the environment. auto env = getEnv(); @@ -422,12 +443,45 @@ static void main_nix_build(int argc, char * * argv) } else env[var.first] = var.second; + std::string structuredAttrsRC; + + if (env.count("__json")) { + StorePathSet inputs; + for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { + auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); + for (auto & i : wantedDepOutputs) { + auto o = outputs.at(i); + store->computeFSClosure(*o, inputs); + } + } + + ParsedDerivation parsedDrv( + StorePath(store->parseStorePath(drvInfo.queryDrvPath())), + drv + ); + + if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { + auto json = structAttrs.value(); + structuredAttrsRC = writeStructuredAttrsShell(json); + + auto attrsJSON = (Path) tmpDir + "/.attrs.json"; + writeFile(attrsJSON, json.dump()); + + auto attrsSH = (Path) tmpDir + "/.attrs.sh"; + writeFile(attrsSH, structuredAttrsRC); + + env["NIX_ATTRS_SH_FILE"] = attrsSH; + env["NIX_ATTRS_JSON_FILE"] = attrsJSON; + keepTmp = true; + } + } + /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't lose the current $PATH directories. */ auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, fmt( + std::string rc = fmt( R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s + (keepTmp ? "trap _nix_shell_clean_tmpdir EXIT; " @@ -436,8 +490,12 @@ static void main_nix_build(int argc, char * * argv) "_nix_shell_clean_tmpdir; ") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" - "dontAddDisableDepTrack=1; " - "[ -e $stdenv/setup ] && source $stdenv/setup; " + // always clear PATH. + // when nix-shell is run impure, we rehydrate it with the `p=$PATH` above + "unset PATH;" + "dontAddDisableDepTrack=1;\n" + + structuredAttrsRC + + "\n[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "PATH=%4%:\"$PATH\"; " "SHELL=%5%; " @@ -455,7 +513,9 @@ static void main_nix_build(int argc, char * * argv) shellEscape(dirOf(*shell)), shellEscape(*shell), (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), - envCommand)); + envCommand); + vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); + writeFile(rcfile, rc); Strings envStrs; for (auto & i : env) @@ -484,6 +544,7 @@ static void main_nix_build(int argc, char * * argv) std::vector pathsToBuild; std::vector> pathsToBuildOrdered; + RealisedPath::Set drvsToCopy; std::map> drvMap; @@ -496,13 +557,13 @@ static void main_nix_build(int argc, char * * argv) pathsToBuild.push_back({drvPath, {outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); + drvsToCopy.insert(drvPath); auto i = drvMap.find(drvPath); if (i != drvMap.end()) i->second.second.insert(outputName); - else { + else drvMap[drvPath] = {drvMap.size(), {outputName}}; - } } buildPaths(pathsToBuild); @@ -517,7 +578,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 02ccbe541..841d50fd3 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -54,10 +54,7 @@ static int main_nix_copy_closure(int argc, char ** argv) for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - RealisedPath::Set closure; - RealisedPath::closure(*from, storePaths2, closure); - - copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + copyClosure(*from, *to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes); return 0; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e04954d45..4056d973d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -879,7 +879,7 @@ static void queryJSON(Globals & globals, vector & elems) placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } @@ -1204,37 +1204,6 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) } -static constexpr GenerationNumber prevGen = std::numeric_limits::max(); - - -static void switchGeneration(Globals & globals, GenerationNumber dstGen) -{ - PathLocks lock; - lockProfile(lock, globals.profile); - - auto [gens, curGen] = findGenerations(globals.profile); - - std::optional dst; - for (auto & i : gens) - if ((dstGen == prevGen && i.number < curGen) || - (dstGen >= 0 && i.number == dstGen)) - dst = i; - - if (!dst) { - if (dstGen == prevGen) - throw Error("no generation older than the current (%1%) exists", curGen.value_or(0)); - else - throw Error("generation %1% does not exist", dstGen); - } - - printInfo("switching from generation %1% to %2%", curGen.value_or(0), dst->number); - - if (globals.dryRun) return; - - switchLink(globals.profile, dst->path); -} - - static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) @@ -1243,7 +1212,7 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg throw UsageError("exactly one argument expected"); if (auto dstGen = string2Int(opArgs.front())) - switchGeneration(globals, *dstGen); + switchGeneration(globals.profile, *dstGen, globals.dryRun); else throw UsageError("expected a generation number"); } @@ -1256,7 +1225,7 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) if (opArgs.size() != 0) throw UsageError("no arguments expected"); - switchGeneration(globals, prevGen); + switchGeneration(globals.profile, {}, globals.dryRun); } @@ -1296,12 +1265,12 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { - if(opArgs.front().size() < 2) - throw Error("invalid number of generations ‘%1%’", opArgs.front()); + if (opArgs.front().size() < 2) + throw Error("invalid number of generations '%1%'", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); auto max = string2Int(str_max); if (!max || *max == 0) - throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); + throw Error("invalid number of generations to keep '%1%'", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set gens; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 5ceb2ae67..1fd4bcbd3 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -131,9 +131,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.forceValue(topLevel); PathSet context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context)); + auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(*aDrvPath.pos, *aDrvPath.value, context)); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context); + Path topLevelOut = state.coerceToPath(*aOutPath.pos, *aOutPath.value, context); /* Realise the resulting store expression. */ debug("building user environment"); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 95903d882..19a954ddd 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -50,9 +50,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context); + printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); else { if (strict) state.forceValueDeep(vRes); std::cout << vRes << std::endl; @@ -153,8 +153,9 @@ static int main_nix_instantiate(int argc, char * * argv) settings.readOnlyMode = true; auto store = openStore(); + auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.searchPath, store); + auto state = std::make_unique(myArgs.searchPath, evalStore, store); state->repair = repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b327793e7..f0ce0368a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -801,6 +801,9 @@ static void opServe(Strings opFlags, Strings opArgs) settings.enforceDeterminism = readInt(in); settings.runDiffHook = true; } + if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { + settings.keepFailed = (bool) readInt(in); + } settings.printRepeatedBuilds = false; }; diff --git a/src/nix/app.cc b/src/nix/app.cc index cf147c631..9719a65dd 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -3,34 +3,79 @@ #include "eval-inline.hh" #include "eval-cache.hh" #include "names.hh" +#include "command.hh" namespace nix { -App Installable::toApp(EvalState & state) +struct InstallableDerivedPath : Installable +{ + ref store; + const DerivedPath derivedPath; + + InstallableDerivedPath(ref store, const DerivedPath & derivedPath) + : store(store) + , derivedPath(derivedPath) + { + } + + + std::string what() override { return derivedPath.to_string(*store); } + + DerivedPaths toDerivedPaths() override + { + return {derivedPath}; + } + + std::optional getStorePath() override + { + return std::nullopt; + } +}; + +/** + * Return the rewrites that are needed to resolve a string whose context is + * included in `dependencies` + */ +StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +{ + StringPairs res; + for (auto & dep : dependencies) + if (auto drvDep = std::get_if(&dep)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + downstreamPlaceholder(store, drvDep->drvPath, outputName), + store.printStorePath(outputPath) + ); + return res; +} + +/** + * Resolve the given string assuming the given context + */ +std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +{ + auto rewrites = resolveRewrites(store, dependencies); + return rewriteStrings(toResolve, rewrites); +} + +UnresolvedApp Installable::toApp(EvalState & state) { auto [cursor, attrPath] = getCursor(state); auto type = cursor->getAttr("type")->getString(); - auto checkProgram = [&](const Path & program) - { - if (!state.store->isInStore(program)) - throw Error("app program '%s' is not in the Nix store", program); - }; - if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - checkProgram(program); std::vector context2; for (auto & [path, name] : context) context2.push_back({state.store->parseStorePath(path), {name}}); - return App { + return UnresolvedApp{App { .context = std::move(context2), .program = program, - }; + }}; } else if (type == "derivation") { @@ -45,15 +90,33 @@ App Installable::toApp(EvalState & state) ? aMainProgram->getString() : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; - checkProgram(program); - return App { + return UnresolvedApp { App { .context = { { drvPath, {outputName} } }, .program = program, - }; + }}; } else throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); } +// FIXME: move to libcmd +App UnresolvedApp::resolve(ref evalStore, ref store) +{ + auto res = unresolved; + + std::vector> installableContext; + + for (auto & ctxElt : unresolved.context) + installableContext.push_back( + std::make_shared(store, ctxElt.toDerivedPath())); + + auto builtContext = build(evalStore, store, Realise::Outputs, installableContext); + res.program = resolveString(*store, unresolved.program, builtContext); + if (!store->isInStore(res.program)) + throw Error("app program '%s' is not in the Nix store", res.program); + + return res; +} + } diff --git a/src/nix/build.cc b/src/nix/build.cc index 03159b6cc..6e31757a2 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,7 +52,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile void run(ref store) override { - auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); + auto buildables = build( + getEvalStore(), store, + dryRun ? Realise::Derivation : Realise::Outputs, + installables, buildMode); + + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); if (dryRun) return; @@ -61,14 +66,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](const BuiltPath::Opaque & bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](DerivedPathWithHints::Built bfd) { - auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); - for (auto & output : builtOutputs) { + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); @@ -79,8 +83,6 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile } updateProfile(buildables); - - if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); } }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 53dccc63a..aca024bca 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -59,7 +59,7 @@ struct CmdBundle : InstallableCommand Strings getDefaultFlakeAttrPathPrefixes() override { - Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + Strings res{"apps." + settings.thisSystem.get() + "."}; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) res.push_back(s); return res; @@ -69,8 +69,7 @@ struct CmdBundle : InstallableCommand { auto evalState = getEvalState(); - auto app = installable->toApp(*evalState); - store->buildPaths(toDerivedPaths(app.context)); + auto app = installable->toApp(*evalState).resolve(getEvalStore(), store); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f59f7c76b..197c85316 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : RealisedPathsCommand +struct CmdCopy : BuiltPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : RealisedPathsCommand SubstituteFlag substitute = NoSubstitute; - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; CmdCopy() - : RealisedPathsCommand(true) + : BuiltPathsCommand(true) { addFlag({ .longName = "from", @@ -75,16 +75,22 @@ struct CmdCopy : RealisedPathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - RealisedPathsCommand::run(store); + BuiltPathsCommand::run(store); } - void run(ref srcStore, std::vector paths) override + void run(ref srcStore, BuiltPaths && paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + RealisedPath::Set stuffToCopy; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*srcStore); + stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); + } + copyPaths( - srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), - NoRepair, checkSigs, substitute); + *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 2cf2a04c9..6a40a0bd3 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -156,9 +156,6 @@ static void daemonLoop() if (chdir("/") == -1) throw SysError("cannot change current directory"); - // Get rid of children automatically; don't let them become zombies. - setSigChldAction(true); - AutoCloseFD fdSocket; // Handle socket-based activation by systemd. @@ -176,6 +173,9 @@ static void daemonLoop() fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } + // Get rid of children automatically; don't let them become zombies. + setSigChldAction(true); + // Loop accepting connections. while (1) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 498a7b45c..5aad53919 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -7,8 +7,10 @@ #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" +#include "run.hh" -#include +#include +#include using namespace nix; @@ -25,94 +27,142 @@ static DevelopSettings developSettings; static GlobalConfig::Register rDevelopSettings(&developSettings); -struct Var -{ - bool exported = true; - bool associative = false; - std::string quoted; // quoted string or array -}; - struct BuildEnvironment { - std::map env; - std::string bashFunctions; -}; + struct String + { + bool exported; + std::string value; -BuildEnvironment readEnvironment(const Path & path) -{ - BuildEnvironment res; + bool operator == (const String & other) const + { + return exported == other.exported && value == other.value; + } + }; - std::set exported; + using Array = std::vector; - debug("reading environment file '%s'", path); + using Associative = std::map; - auto file = readFile(path); + using Value = std::variant; - auto pos = file.cbegin(); + std::map vars; + std::map bashFunctions; - static std::string varNameRegex = - R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + static BuildEnvironment fromJSON(std::string_view in) + { + BuildEnvironment res; - static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + std::set exported; - static std::string dquotedStringRegex = - R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + auto json = nlohmann::json::parse(in); - static std::string squotedStringRegex = - R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; - - static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; - - static std::regex declareRegex( - "^declare -a?x (" + varNameRegex + ")(=(" + - dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); - - static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); - - /* Note: we distinguish between an indexed and associative array - using the space before the closing parenthesis. Will - undoubtedly regret this some day. */ - static std::regex assocArrayRegex( - "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); - - static std::regex functionRegex( - "^" + varNameRegex + " \\(\\) *\n"); - - while (pos != file.end()) { - - std::smatch match; - - if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - exported.insert(match[1]); + for (auto & [name, info] : json["variables"].items()) { + std::string type = info["type"]; + if (type == "var" || type == "exported") + res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }}); + else if (type == "array") + res.vars.insert({name, (Array) info["value"]}); + else if (type == "associative") + res.vars.insert({name, (Associative) info["value"]}); } - else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }}); + for (auto & [name, def] : json["bashFunctions"].items()) { + res.bashFunctions.insert({name, def}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }}); - } - - else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { - res.bashFunctions = std::string(pos, file.cend()); - break; - } - - else throw Error("shell environment '%s' has unexpected line '%s'", - path, file.substr(pos - file.cbegin(), 60)); + return res; } - res.env.erase("__output"); + std::string toJSON() const + { + auto res = nlohmann::json::object(); - return res; -} + auto vars2 = nlohmann::json::object(); + for (auto & [name, value] : vars) { + auto info = nlohmann::json::object(); + if (auto str = std::get_if(&value)) { + info["type"] = str->exported ? "exported" : "var"; + info["value"] = str->value; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "array"; + info["value"] = *arr; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "associative"; + info["value"] = *arr; + } + vars2[name] = std::move(info); + } + res["variables"] = std::move(vars2); + + res["bashFunctions"] = bashFunctions; + + auto json = res.dump(); + + assert(BuildEnvironment::fromJSON(json) == *this); + + return json; + } + + void toBash(std::ostream & out, const std::set & ignoreVars) const + { + for (auto & [name, value] : vars) { + if (!ignoreVars.count(name)) { + if (auto str = std::get_if(&value)) { + out << fmt("%s=%s\n", name, shellEscape(str->value)); + if (str->exported) + out << fmt("export %s\n", name); + } + else if (auto arr = std::get_if(&value)) { + out << "declare -a " << name << "=("; + for (auto & s : *arr) + out << shellEscape(s) << " "; + out << ")\n"; + } + else if (auto arr = std::get_if(&value)) { + out << "declare -A " << name << "=("; + for (auto & [n, v] : *arr) + out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << ")\n"; + } + } + } + + for (auto & [name, def] : bashFunctions) { + out << name << " ()\n{\n" << def << "}\n"; + } + } + + static std::string getString(const Value & value) + { + if (auto str = std::get_if(&value)) + return str->value; + else + throw Error("bash variable is not a string"); + } + + static Array getStrings(const Value & value) + { + if (auto str = std::get_if(&value)) + return tokenizeString(str->value); + else if (auto arr = std::get_if(&value)) { + return *arr; + } else if (auto assoc = std::get_if(&value)) { + Array assocKeys; + std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); }); + return assocKeys; + } + else + throw Error("bash variable is not a string or array"); + } + + bool operator == (const BuildEnvironment & other) const + { + return vars == other.vars && bashFunctions == other.bashFunctions; + } +}; const static std::string getEnvSh = #include "get-env.sh.gen.hh" @@ -123,15 +173,15 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) +static StorePath getDerivationEnvironment(ref store, ref evalStore, const StorePath & drvPath) { - auto drv = store->derivationFromPath(drvPath); + auto drv = evalStore->derivationFromPath(drvPath); auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); drv.args = {store->printStorePath(getEnvShPath)}; @@ -144,26 +194,34 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; - for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; - drv.env[output.first] = ""; - } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = hashPlaceholder(output.first); + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); - for (auto & output : drv.outputs) { - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; - drv.env[output.first] = store->printStorePath(outPath); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } } - auto shellDrvPath = writeDerivation(*store, drv); + auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); - for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { - auto & [_1, optPath] = outputAndOptPath; + for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); @@ -177,18 +235,15 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) struct Common : InstallableCommand, MixProfile { - std::set ignoreVars{ + std::set ignoreVars{ "BASHOPTS", - "EUID", "HOME", // FIXME: don't ignore in pure mode? - "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", + "NIX_REMOTE", "PPID", - "PWD", "SHELLOPTS", - "SHLVL", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", @@ -224,22 +279,10 @@ struct Common : InstallableCommand, MixProfile out << "nix_saved_PATH=\"$PATH\"\n"; - for (auto & i : buildEnvironment.env) { - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted); - else { - out << fmt("%s=%s\n", i.first, i.second.quoted); - if (i.second.exported) - out << fmt("export %s\n", i.first); - } - } - } + buildEnvironment.toBash(out, ignoreVars); out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); @@ -249,25 +292,25 @@ struct Common : InstallableCommand, MixProfile auto script = out.str(); /* Substitute occurrences of output paths. */ - auto outputs = buildEnvironment.env.find("outputs"); - assert(outputs != buildEnvironment.env.end()); + auto outputs = buildEnvironment.vars.find("outputs"); + assert(outputs != buildEnvironment.vars.end()); // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : tokenizeString>(replaceStrings(outputs->second.quoted, "'", ""))) { - auto from = buildEnvironment.env.find(outputName); - assert(from != buildEnvironment.env.end()); + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); // FIXME: unquote - rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); } /* Substitute redirects. */ for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toDerivedPathWithHints(); - auto doRedirect = [&](const StorePath & path) - { + auto builtPaths = toStorePaths( + getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); + for (auto & path: builtPaths) { auto from = store->printStorePath(path); if (script.find(from) == std::string::npos) warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); @@ -275,16 +318,7 @@ struct Common : InstallableCommand, MixProfile printInfo("redirecting '%s' to '%s'", from, dir); rewrites.insert({from, dir}); } - }; - std::visit(overloaded { - [&](const DerivedPathWithHints::Opaque & bo) { - doRedirect(bo.path); - }, - [&](const DerivedPathWithHints::Built & bfd) { - for (auto & [outputName, path] : bfd.outputs) - if (path) doRedirect(*path); - }, - }, buildable.raw()); + } } return rewriteStrings(script, rewrites); @@ -294,6 +328,12 @@ struct Common : InstallableCommand, MixProfile { return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; } + Strings getDefaultFlakeAttrPathPrefixes() override + { + auto res = SourceExprCommand::getDefaultFlakeAttrPathPrefixes(); + res.emplace_front("devShells." + settings.thisSystem.get() + "."); + return res; + } StorePath getShellOutPath(ref store) { @@ -309,7 +349,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, drvPath); + return getDerivationEnvironment(store, getEvalStore(), drvPath); } } @@ -321,7 +361,9 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {readEnvironment(strPath), strPath}; + debug("reading environment file '%s'", strPath); + + return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath}; } }; @@ -350,6 +392,12 @@ struct CmdDevelop : Common, MixEnvironment .handler = {&phase}, }); + addFlag({ + .longName = "unpack", + .description = "Run the `unpack` phase.", + .handler = {&phase, {"unpack"}}, + }); + addFlag({ .longName = "configure", .description = "Run the `configure` phase.", @@ -404,7 +452,7 @@ struct CmdDevelop : Common, MixEnvironment if (verbosity >= lvlDebug) script += "set -x\n"; - script += fmt("rm -f '%s'\n", rcFilePath); + script += fmt("command rm -f '%s'\n", rcFilePath); if (phase) { if (!command.empty()) @@ -423,7 +471,7 @@ struct CmdDevelop : Common, MixEnvironment } else { - script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); if (developSettings.bashPromptSuffix != "") @@ -432,8 +480,6 @@ struct CmdDevelop : Common, MixEnvironment writeFull(rcFileFd.get(), script); - stopProgressBar(); - setEnviron(); // prevent garbage collection until shell exits setenv("NIX_GCROOT", gcroot.data(), 1); @@ -443,16 +489,20 @@ struct CmdDevelop : Common, MixEnvironment try { auto state = getEvalState(); + auto nixpkgsLockFlags = lockFlags; + nixpkgsLockFlags.inputOverrides = {}; + nixpkgsLockFlags.inputUpdates = {}; + auto bashInstallable = std::make_shared( this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, - lockFlags); + nixpkgsLockFlags); - shell = state->store->printStorePath( - toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + shell = store->printStorePath( + toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; } catch (Error &) { ignoreException(); } @@ -462,15 +512,25 @@ struct CmdDevelop : Common, MixEnvironment auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; - restoreProcessContext(); + // Need to chdir since phases assume in flake directory + if (phase) { + // chdir if installable is a flake of type git+file or path + auto installableFlake = std::dynamic_pointer_cast(installable); + if (installableFlake) { + auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); + if (sourcePath) { + if (chdir(sourcePath->c_str()) == -1) { + throw SysError("chdir to '%s' failed", *sourcePath); + } + } + } + } - execvp(shell.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing shell '%s'", shell); + runProgramInStore(store, shell, args); } }; -struct CmdPrintDevEnv : Common +struct CmdPrintDevEnv : Common, MixJSON { std::string description() override { @@ -492,7 +552,10 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(store, buildEnvironment); + logger->writeToStdout( + json + ? buildEnvironment.toJSON() + : makeRcScript(store, buildEnvironment)); } }; diff --git a/src/nix/develop.md b/src/nix/develop.md index e71d9f8aa..1f214966a 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -29,6 +29,7 @@ R""( * Run a particular build phase directly: ```console + # nix develop --unpack # nix develop --configure # nix develop --build # nix develop --check @@ -84,11 +85,20 @@ the flake's `nixConfig` attribute. # Flake output attributes -If no flake output attribute is given, `nix run` tries the following +If no flake output attribute is given, `nix develop` tries the following flake output attributes: * `devShell.` * `defaultPackage.` +If a flake output *name* is given, `nix develop` tries the following flake +output attributes: + +* `devShells..` + +* `packages..` + +* `legacyPackages..` + )"" diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0c7d531c1..734c41e0e 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -131,9 +131,9 @@ struct CmdDiffClosures : SourceExprCommand void run(ref store) override { auto before = parseInstallable(store, _before); - auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before); + auto beforePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, before); auto after = parseInstallable(store, _after); - auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after); + auto afterPath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, after); printClosureDiff(store, beforePath, afterPath, ""); } }; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index b26417b18..fc48db0d7 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -31,7 +31,7 @@ struct CmdEdit : InstallableCommand auto [v, pos] = installable->toValue(*state); try { - pos = findDerivationFilename(*state, *v, installable->what()); + pos = findPackageFilename(*state, *v, installable->what()); } catch (NoPositionInfo &) { } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 65d61e005..c7517cf79 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -112,7 +112,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context); } else { diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index dc079ba0c..07031c909 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -22,42 +22,47 @@ This command verifies that the flake specified by flake reference that the derivations specified by the flake's `checks` output can be built successfully. +If the `keep-going` option is set to `true`, Nix will keep evaluating as much +as it can and report the errors as it encounters them. Otherwise it will stop +at the first error. + # Evaluation checks -This following flake output attributes must be derivations: +The following flake output attributes must be derivations: * `checks.`*system*`.`*name* -* `defaultPackage.`*system*` -* `devShell.`*system*` -* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `defaultPackage.`*system* +* `devShell.`*system* +* `devShells.`*system*`.`*name* +* `nixosConfigurations.`*name*`.config.system.build.toplevel` * `packages.`*system*`.`*name* The following flake output attributes must be [app definitions](./nix3-run.md): * `apps.`*system*`.`*name* -* `defaultApp.`*system*` +* `defaultApp.`*system* The following flake output attributes must be [template definitions](./nix3-flake-init.md): * `defaultTemplate` -* `templates`.`*name* +* `templates.`*name* The following flake output attributes must be *Nixpkgs overlays*: * `overlay` -* `overlays`.`*name* +* `overlays.`*name* The following flake output attributes must be *NixOS modules*: * `nixosModule` -* `nixosModules`.`*name* +* `nixosModules.`*name* The following flake output attributes must be [bundlers](./nix3-bundle.md): -* `bundlers`.`*name* +* `bundlers.`*name* * `defaultBundler` In addition, the `hydraJobs` output is evaluated in the same way as diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md index 1a42c44a0..e484cf47e 100644 --- a/src/nix/flake-show.md +++ b/src/nix/flake-show.md @@ -35,4 +35,7 @@ specified by flake reference *flake-url*. These are the top-level attributes in the `outputs` of the flake, as well as lower-level attributes for some standard outputs (e.g. `packages` or `checks`). +With `--json`, the output is in a JSON representation suitable for automatic +processing by other tools. + )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 62a413e27..97f4d911c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -84,6 +84,7 @@ struct CmdFlakeUpdate : FlakeCommand lockFlags.recreateLockFile = true; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -114,6 +115,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -129,8 +131,18 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, state.forceAttrs(*aOutputs->value); - for (auto & attr : *aOutputs->value->attrs) - callback(attr.name, *attr.value, *attr.pos); + auto sHydraJobs = state.symbols.create("hydraJobs"); + + /* Hack: ensure that hydraJobs is evaluated before anything + else. This way we can disable IFD for hydraJobs and then enable + it for other outputs. */ + if (auto attr = aOutputs->value->attrs->get(sHydraJobs)) + callback(attr->name, *attr->value, *attr->pos); + + for (auto & attr : *aOutputs->value->attrs) { + if (attr.name != sHydraJobs) + callback(attr.name, *attr.value, *attr.pos); + } } struct CmdFlakeMetadata : FlakeCommand, MixJSON @@ -240,6 +252,14 @@ struct CmdFlakeInfo : CmdFlakeMetadata } }; +static bool argHasName(std::string_view arg, std::string_view expected) +{ + return + arg == expected + || arg == "_" + || (hasPrefix(arg, "_") && arg.substr(1) == expected); +} + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -267,30 +287,50 @@ struct CmdFlakeCheck : FlakeCommand void run(nix::ref store) override { - settings.readOnlyMode = !build; + if (!build) { + settings.readOnlyMode = true; + evalSettings.enableImportFromDerivation.setDefault(false); + } auto state = getEvalState(); + + lockFlags.applyNixConfig = true; auto flake = lockFlake(); + bool hasErrors = false; + auto reportError = [&](const Error & e) { + try { + throw e; + } catch (Error & e) { + if (settings.keepGoing) { + ignoreException(); + hasErrors = true; + } + else + throw; + } + }; + // FIXME: rewrite to use EvalCache. auto checkSystemName = [&](const std::string & system, const Pos & pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) - throw Error("'%s' is not a valid system type, at %s", system, pos); + reportError(Error("'%s' is not a valid system type, at %s", system, pos)); }; - auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional { try { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); // FIXME: check meta attributes - return store->parseStorePath(drvInfo->queryDrvPath()); + return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); - throw; + reportError(e); } + return std::nullopt; }; std::vector drvPaths; @@ -307,23 +347,27 @@ struct CmdFlakeCheck : FlakeCommand #endif } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); - throw; + reportError(e); } }; auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() + || v.lambda.fun->hasFormals() + || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); - if (!body || body->matchAttrs || std::string(body->arg) != "prev") + if (!body + || body->hasFormals() + || !argHasName(body->arg, "prev")) throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); - throw; + reportError(e); } }; @@ -331,7 +375,7 @@ struct CmdFlakeCheck : FlakeCommand try { state->forceValue(v, pos); if (v.isLambda()) { - if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); } else if (v.type() == nAttrs) { for (auto & attr : *v.attrs) @@ -347,7 +391,7 @@ struct CmdFlakeCheck : FlakeCommand // check the module. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); - throw; + reportError(e); } }; @@ -362,14 +406,18 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *v.attrs) { state->forceAttrs(*attr.value, *attr.pos); - if (!state->isDerivation(*attr.value)) - checkHydraJobs(attrPath + "." + (std::string) attr.name, - *attr.value, *attr.pos); + auto attrPath2 = attrPath + "." + (std::string) attr.name; + if (state->isDerivation(*attr.value)) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking Hydra job '%s'", attrPath2)); + checkDerivation(attrPath2, *attr.value, *attr.pos); + } else + checkHydraJobs(attrPath2, *attr.value, *attr.pos); } } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); - throw; + reportError(e); } }; @@ -384,7 +432,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); - throw; + reportError(e); } }; @@ -418,7 +466,7 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + reportError(e); } }; @@ -428,12 +476,12 @@ struct CmdFlakeCheck : FlakeCommand if (!v.isLambda()) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || - v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || - v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + !v.lambda.fun->formals->argNames.count(state->symbols.create("program")) || + !v.lambda.fun->formals->argNames.count(state->symbols.create("system"))) throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + reportError(e); } }; @@ -450,6 +498,8 @@ struct CmdFlakeCheck : FlakeCommand fmt("checking flake output '%s'", name)); try { + evalSettings.enableImportFromDerivation.setDefault(name != "hydraJobs"); + state->forceValue(vOutput, pos); if (name == "checks") { @@ -461,13 +511,13 @@ struct CmdFlakeCheck : FlakeCommand auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); - if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{drvPath}); + if (drvPath && (std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back(DerivedPath::Built{*drvPath}); } } } - else if (name == "packages") { + else if (name == "packages" || name == "devShells") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); @@ -574,7 +624,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); - throw; + reportError(e); } }); } @@ -583,6 +633,8 @@ struct CmdFlakeCheck : FlakeCommand Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); store->buildPaths(drvPaths); } + if (hasErrors) + throw Error("some errors were encountered during the evaluation"); } }; @@ -820,12 +872,12 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (!dryRun && !dstUri.empty()) { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(store, dstStore, sources); + copyPaths(*store, *dstStore, sources); } } }; -struct CmdFlakeShow : FlakeCommand +struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; @@ -852,51 +904,69 @@ struct CmdFlakeShow : FlakeCommand void run(nix::ref store) override { + evalSettings.enableImportFromDerivation.setDefault(false); + auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); - std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; + std::function & attrPath, + const std::string & headerPrefix, + const std::string & nextPrefix)> visit; - visit = [&](eval_cache::AttrCursor & visitor, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) + visit = [&]( + eval_cache::AttrCursor & visitor, + const std::vector & attrPath, + const std::string & headerPrefix, + const std::string & nextPrefix) + -> nlohmann::json { + auto j = nlohmann::json::object(); + Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); try { auto recurse = [&]() { - logger->cout("%s", headerPrefix); + if (!json) + logger->cout("%s", headerPrefix); auto attrs = visitor.getAttrs(); for (const auto & [i, attr] : enumerate(attrs)) { bool last = i + 1 == attrs.size(); auto visitor2 = visitor.getAttr(attr); auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*visitor2, attrPath2, + auto j2 = visit(*visitor2, attrPath2, fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), nextPrefix + (last ? treeNull : treeLine)); + if (json) j.emplace(attr, std::move(j2)); } }; auto showDerivation = [&]() { auto name = visitor.getAttr(state->sName)->getString(); - - /* - std::string description; - - if (auto aMeta = visitor.maybeGetAttr("meta")) { - if (auto aDescription = aMeta->maybeGetAttr("description")) - description = aDescription->getString(); + if (json) { + std::optional description; + if (auto aMeta = visitor.maybeGetAttr("meta")) { + if (auto aDescription = aMeta->maybeGetAttr("description")) + description = aDescription->getString(); + } + j.emplace("type", "derivation"); + j.emplace("name", name); + if (description) + j.emplace("description", *description); + } else { + logger->cout("%s: %s '%s'", + headerPrefix, + attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : + attrPath.size() >= 2 && attrPath[0] == "devShells" ? "development environment" : + attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : + attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : + "package", + name); } - */ - - logger->cout("%s: %s '%s'", - headerPrefix, - attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : - attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : - attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : - "package", - name); }; if (attrPath.size() == 0 @@ -911,6 +981,7 @@ struct CmdFlakeShow : FlakeCommand || ((attrPath.size() == 1 || attrPath.size() == 2) && (attrPath[0] == "checks" || attrPath[0] == "packages" + || attrPath[0] == "devShells" || attrPath[0] == "apps")) ) { @@ -919,7 +990,7 @@ struct CmdFlakeShow : FlakeCommand else if ( (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) - || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages")) + || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) ) { if (visitor.isDerivation()) @@ -939,7 +1010,7 @@ struct CmdFlakeShow : FlakeCommand if (attrPath.size() == 1) recurse(); else if (!showLegacy) - logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); + logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); else { if (visitor.isDerivation()) showDerivation(); @@ -956,7 +1027,11 @@ struct CmdFlakeShow : FlakeCommand auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") throw EvalError("not an app definition"); - logger->cout("%s: app", headerPrefix); + if (json) { + j.emplace("type", "app"); + } else { + logger->cout("%s: app", headerPrefix); + } } else if ( @@ -964,27 +1039,41 @@ struct CmdFlakeShow : FlakeCommand (attrPath.size() == 2 && attrPath[0] == "templates")) { auto description = visitor.getAttr("description")->getString(); - logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + if (json) { + j.emplace("type", "template"); + j.emplace("description", description); + } else { + logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + } } else { - logger->cout("%s: %s", - headerPrefix, + auto [type, description] = (attrPath.size() == 1 && attrPath[0] == "overlay") - || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" : - attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : - attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" : - ANSI_YELLOW "unknown" ANSI_NORMAL); + || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : + attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : + (attrPath.size() == 1 && attrPath[0] == "nixosModule") + || (attrPath.size() == 2 && attrPath[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : + std::make_pair("unknown", "unknown"); + if (json) { + j.emplace("type", type); + } else { + logger->cout("%s: " ANSI_WARNING "%s" ANSI_NORMAL, headerPrefix, description); + } } } catch (EvalError & e) { if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) throw; } + + return j; }; auto cache = openEvalCache(*state, flake); - visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + auto j = visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + if (json) + logger->cout("%s", j.dump()); } }; @@ -1062,7 +1151,7 @@ struct CmdFlake : NixMultiCommand { if (!command) throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature("flakes"); + settings.requireExperimentalFeature(Xp::Flakes); command->second->prepare(); command->second->run(); } diff --git a/src/nix/flake.md b/src/nix/flake.md index 9e936a049..3b5812a0a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -186,8 +186,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` - or `.tar.bz2`. + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, + `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: @@ -225,7 +225,7 @@ Currently the `type` attribute can be one of the following: [flake:](/(/rev)?)? ``` - These perform a lookup of `` in the flake registry. or + These perform a lookup of `` in the flake registry. For example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake references. The specified `rev` and/or `ref` are merged with the entry in the registry; see [nix registry](./nix3-registry.md) for diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 091c0f573..42c806450 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -8,12 +8,123 @@ if [[ -n $stdenv ]]; then source $stdenv/setup fi -for __output in $outputs; do +# Better to use compgen, but stdenv bash doesn't have it. +__vars="$(declare -p)" +__functions="$(declare -F)" + +__dumpEnv() { + printf '{\n' + + printf ' "bashFunctions": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi + __fun_name="${BASH_REMATCH[1]}" + __fun_body="$(type $__fun_name)" + if [[ $__fun_body =~ \{(.*)\} ]]; then + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + __fun_body="${BASH_REMATCH[1]}" + printf " " + __escapeString "$__fun_name" + printf ':' + __escapeString "$__fun_body" + else + printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2 + return 1 + fi + done < <(printf "%s\n" "$__functions") + printf '\n },\n' + + printf ' "variables": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi + local type="${BASH_REMATCH[1]}" + local __var_name="${BASH_REMATCH[2]}" + + if [[ $__var_name =~ ^BASH_ || \ + $__var_name = _ || \ + $__var_name = DIRSTACK || \ + $__var_name = EUID || \ + $__var_name = FUNCNAME || \ + $__var_name = HISTCMD || \ + $__var_name = HOSTNAME || \ + $__var_name = GROUPS || \ + $__var_name = PIPESTATUS || \ + $__var_name = PWD || \ + $__var_name = RANDOM || \ + $__var_name = SHLVL || \ + $__var_name = SECONDS \ + ]]; then continue; fi + + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + + printf " " + __escapeString "$__var_name" + printf ': {' + + # FIXME: handle -i, -r, -n. + if [[ $type == -x ]]; then + printf '"type": "exported", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -- ]]; then + printf '"type": "var", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -a ]]; then + printf '"type": "array", "value": [' + local __first2=1 + __var_name="$__var_name[@]" + for __i in "${!__var_name}"; do + if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi + __escapeString "$__i" + printf ' ' + done + printf ']' + elif [[ $type == -A ]]; then + printf '"type": "associative", "value": {\n' + local __first2=1 + declare -n __var_name2="$__var_name" + for __i in "${!__var_name2[@]}"; do + if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi + printf " " + __escapeString "$__i" + printf ": " + __escapeString "${__var_name2[$__i]}" + done + printf '\n }' + else + printf '"type": "unknown"' + fi + + printf "}" + done < <(printf "%s\n" "$__vars") + printf '\n }\n}' +} + +__escapeString() { + local __s="$1" + __s="${__s//\\/\\\\}" + __s="${__s//\"/\\\"}" + __s="${__s//$'\n'/\\n}" + __s="${__s//$'\r'/\\r}" + __s="${__s//$'\t'/\\t}" + printf '"%s"' "$__s" +} + +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` +# must contain the array's keys (hence `${!...[@]}`) in this case. +if [ -e .attrs.sh ]; then + __olist="${!outputs[@]}" +else + __olist=$outputs +fi + +for __output in $__olist; do if [[ -z $__done ]]; then - export > ${!__output} - set >> ${!__output} + __dumpEnv > ${!__output} __done=1 else - echo -n >> ${!__output} + echo -n >> "${!__output}" fi done diff --git a/src/nix/local.mk b/src/nix/local.mk index 83b6dd08b..e4ec7634d 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -14,7 +14,7 @@ nix_SOURCES := \ $(wildcard src/nix-instantiate/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -I doc/manual nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd @@ -30,3 +30,5 @@ src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh + +src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc index 638bb5073..fd3c1d787 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toDerivedPathWithHints(); + auto b = installable->toDerivedPath(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](const DerivedPath::Opaque & bo) { return sub->getBuildLog(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](const DerivedPath::Built & bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b.raw()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 008482be3..60b0aa410 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -10,6 +10,7 @@ #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" +#include "markdown.hh" #include #include @@ -163,9 +164,46 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs } }; -static void showHelp(std::vector subcommand) +/* Render the help for the specified subcommand to stdout using + lowdown. */ +static void showHelp(std::vector subcommand, MultiCommand & toplevel) { - showManPage(subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand))); + auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); + + evalSettings.restrictEval = false; + evalSettings.pureEval = false; + EvalState state({}, openStore("dummy://")); + + auto vGenerateManpage = state.allocValue(); + state.eval(state.parseExprFromString( + #include "generate-manpage.nix.gen.hh" + , "/"), *vGenerateManpage); + + auto vUtils = state.allocValue(); + state.cacheFile( + "/utils.nix", "/utils.nix", + state.parseExprFromString( + #include "utils.nix.gen.hh" + , "/"), + *vUtils); + + auto vArgs = state.allocValue(); + state.mkAttrs(*vArgs, 16); + auto vJson = state.allocAttr(*vArgs, state.symbols.create("command")); + mkString(*vJson, toplevel.toJSON().dump()); + vArgs->attrs->sort(); + + auto vRes = state.allocValue(); + state.callFunction(*vGenerateManpage, *vArgs, *vRes, noPos); + + auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); + if (!attr) + throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); + + auto markdown = state.forceString(*attr->value); + + RunPager pager; + std::cout << renderMarkdownToTerminal(markdown) << "\n"; } struct CmdHelp : Command @@ -194,7 +232,10 @@ struct CmdHelp : Command void run() override { - showHelp(subcommand); + assert(parent); + MultiCommand * toplevel = parent; + while (toplevel->parent) toplevel = toplevel->parent; + showHelp(subcommand, *toplevel); } }; @@ -214,6 +255,16 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + #if __linux__ + if (getuid() == 0) { + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } catch (Error & e) { } + } + #endif + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); @@ -277,7 +328,7 @@ void mainWrapped(int argc, char * * argv) } else break; } - showHelp(subcommand); + showHelp(subcommand, args); return; } catch (UsageError &) { if (!completions) throw; @@ -296,7 +347,7 @@ void mainWrapped(int argc, char * * argv) if (args.command->first != "repl" && args.command->first != "doctor" && args.command->first != "upgrade-nix") - settings.requireExperimentalFeature("nix-command"); + settings.requireExperimentalFeature(Xp::NixCommand); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f5bdc7e65..12f303a10 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -25,7 +25,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ; } - void run(ref store, StorePaths storePaths) override + void run(ref store, StorePaths && storePaths) override { auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 518cd5568..3743d7504 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -79,7 +79,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << fmt("\t%6.1f%c", res, idents.at(power)); } - void run(ref store, StorePaths storePaths) override + void run(ref store, StorePaths && storePaths) override { size_t pathLen = 0; for (auto & storePath : storePaths) diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 76a83e39d..7a1714ba4 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -82,7 +82,7 @@ This command shows information about the store paths produced by By default, this command only prints the store paths. You can get additional information by passing flags such as `--closure-size`, ---size`, `--sigs` or `--json`. +`--size`, `--sigs` or `--json`. > **Warning** > diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b7da3ea5a..768d37595 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -199,26 +199,24 @@ static int main_nix_prefetch_url(int argc, char * * argv) state->forceAttrs(v); /* Extract the URL. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) + auto & attr = v.attrs->need(state->symbols.create("urls")); + state->forceList(*attr.value); + if (attr.value->listSize() < 1) throw Error("'urls' list is empty"); - url = state->forceString(*attr->value->listElems()[0]); + url = state->forceString(*attr.value->listElems()[0]); /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) + auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); + if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state->forceString(*attr->value) == "recursive"; + unpack = state->forceString(*attr2->value) == "recursive"; /* Extract the name. */ if (!name) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); + auto attr3 = v.attrs->get(state->symbols.create("name")); + if (!attr3) + name = state->forceString(*attr3->value); } } @@ -283,8 +281,6 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON expectArg("url", &url); } - Category category() override { return catUtility; } - std::string description() override { return "download a file into the Nix store"; diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index b80252acf..2aad491de 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -8,12 +8,43 @@ R""( # . <(nix print-dev-env nixpkgs#hello) ``` +* Get the build environment in JSON format: + + ```console + # nix print-dev-env nixpkgs#hello --json + ``` + + The output will look like this: + + ```json + { + "bashFunctions": { + "buildPhase": " \n runHook preBuild;\n...", + ... + }, + "variables": { + "src": { + "type": "exported", + "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz" + }, + "postUnpackHooks": { + "type": "array", + "value": ["_updateSourceDateEpochFromSourceRoot"] + }, + ... + } + } + ``` + # Description -This command prints a shell script that can be sourced by `b`ash and -that sets the environment variables and shell functions defined by the -build process of *installable*. This allows you to get a similar build +This command prints a shell script that can be sourced by `bash` and +that sets the variables and shell functions defined by the build +process of *installable*. This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). +With `--json`, the output is a JSON serialisation of the variables and +functions defined by the build process. + )"" diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md index d0fe40c82..f0bfe5037 100644 --- a/src/nix/profile-history.md +++ b/src/nix/profile-history.md @@ -6,10 +6,10 @@ R""( ```console # nix profile history - Version 508 -> 509: + Version 508 (2020-04-10): flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 - Version 509 -> 510: + Version 509 (2020-05-16) <- 508: flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 ``` diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index dcf825da9..ba85441d8 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -15,6 +15,7 @@ R""( ``` * Remove all packages: + ```console # nix profile remove '.*' ``` diff --git a/src/nix/profile-rollback.md b/src/nix/profile-rollback.md new file mode 100644 index 000000000..6bb75aa5e --- /dev/null +++ b/src/nix/profile-rollback.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Roll back your default profile to the previous version: + + ```console + # nix profile rollback + switching profile from version 519 to 518 + ``` + +* Switch your default profile to version 510: + + ```console + # nix profile rollback --to 510 + switching profile from version 518 to 510 + ``` + +# Description + +This command switches a profile to the most recent version older +than the currently active version, or if `--to` *N* is given, to +version *N* of the profile. To see the available versions of a +profile, use `nix profile history`. + +)"" diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 2bd5d256d..e06e74abe 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -18,7 +18,7 @@ R""( * Upgrade a specific profile element by number: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … # nix profile upgrade 0 diff --git a/src/nix/profile-wipe-history.md b/src/nix/profile-wipe-history.md new file mode 100644 index 000000000..b4b262864 --- /dev/null +++ b/src/nix/profile-wipe-history.md @@ -0,0 +1,20 @@ +R""( + +# Examples + +* Delete all versions of the default profile older than 100 days: + + ```console + # nix profile wipe-history --profile /tmp/profile --older-than 100d + removing profile version 515 + removing profile version 514 + ``` + +# Description + +This command deletes non-current versions of a profile, making it +impossible to roll back to these versions. By default, all non-current +versions are deleted. With `--older-than` *N*`d`, all non-current +versions older than *N* days are deleted. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 667904cd2..96a20f673 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -12,6 +12,7 @@ #include #include +#include using namespace nix; @@ -97,10 +98,8 @@ struct ProfileManifest else if (pathExists(profile + "/manifest.nix")) { // FIXME: needed because of pure mode; ugly. - if (state.allowedPaths) { - state.allowedPaths->insert(state.store->followLinksToStore(profile)); - state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); - } + state.allowPath(state.store->followLinksToStore(profile)); + state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); @@ -253,17 +252,17 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile manifest.elements.emplace_back(std::move(element)); } else { - auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + auto buildables = build(getEvalStore(), store, Realise::Outputs, {installable}, bmNormal); for (auto & buildable : buildables) { ProfileElement element; std::visit(overloaded { - [&](DerivedPathWithHints::Opaque bo) { + [&](const BuiltPath::Opaque & bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](DerivedPathWithHints::Built bfd) { + [&](const BuiltPath::Built & bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? @@ -426,7 +425,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {drv.outputName}}); } } @@ -528,10 +527,11 @@ struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile if (!first) std::cout << "\n"; first = false; - if (prevGen) - std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); - else - std::cout << fmt("Version %d:\n", gen.number); + std::cout << fmt("Version %s%d" ANSI_NORMAL " (%s)%s:\n", + gen.number == curGen ? ANSI_GREEN : ANSI_BOLD, + gen.number, + std::put_time(std::gmtime(&gen.creationTime), "%Y-%m-%d"), + prevGen ? fmt(" <- %d", prevGen->first.number) : ""); ProfileManifest::printDiff( prevGen ? prevGen->second : ProfileManifest(), @@ -543,6 +543,76 @@ struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile } }; +struct CmdProfileRollback : virtual StoreCommand, MixDefaultProfile, MixDryRun +{ + std::optional version; + + CmdProfileRollback() + { + addFlag({ + .longName = "to", + .description = "The profile version to roll back to.", + .labels = {"version"}, + .handler = {&version}, + }); + } + + std::string description() override + { + return "roll back to the previous version or a specified version of a profile"; + } + + std::string doc() override + { + return + #include "profile-rollback.md" + ; + } + + void run(ref store) override + { + switchGeneration(*profile, version, dryRun); + } +}; + +struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRun +{ + std::optional minAge; + + CmdProfileWipeHistory() + { + addFlag({ + .longName = "older-than", + .description = + "Delete versions older than the specified age. *age* " + "must be in the format *N*`d`, where *N* denotes a number " + "of days.", + .labels = {"age"}, + .handler = {&minAge}, + }); + } + + std::string description() override + { + return "delete non-current versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-wipe-history.md" + ; + } + + void run(ref store) override + { + if (minAge) + deleteGenerationsOlderThan(*profile, *minAge, dryRun); + else + deleteOldGenerations(*profile, dryRun); + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -553,6 +623,8 @@ struct CmdProfile : NixMultiCommand {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, {"history", []() { return make_ref(); }}, + {"rollback", []() { return make_ref(); }}, + {"wipe-history", []() { return make_ref(); }}, }) { } diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 9ee9ccb91..c9a7157cd 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -28,7 +28,7 @@ struct CmdRealisation : virtual NixMultiCommand static auto rCmdRealisation = registerCommand("realisation"); -struct CmdRealisationInfo : RealisedPathsCommand, MixJSON +struct CmdRealisationInfo : BuiltPathsCommand, MixJSON { std::string description() override { @@ -44,12 +44,19 @@ struct CmdRealisationInfo : RealisedPathsCommand, MixJSON Category category() override { return catSecondary; } - void run(ref store, std::vector paths) override + void run(ref store, BuiltPaths && paths) override { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); + RealisedPath::Set realisations; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*store); + realisations.insert(theseRealisations.begin(), theseRealisations.end()); + } + if (json) { nlohmann::json res = nlohmann::json::array(); - for (auto & path : paths) { + for (auto & path : realisations) { nlohmann::json currentPath; if (auto realisation = std::get_if(&path.raw)) currentPath = realisation->toJSON(); @@ -61,7 +68,7 @@ struct CmdRealisationInfo : RealisedPathsCommand, MixJSON std::cout << res.dump(); } else { - for (auto & path : paths) { + for (auto & path : realisations) { if (auto realisation = std::get_if(&path.raw)) { std::cout << realisation->id.to_string() << " " << diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md index 80a31996a..a947fa0b3 100644 --- a/src/nix/registry-add.md +++ b/src/nix/registry-add.md @@ -21,6 +21,13 @@ R""( # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs ``` +* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake + registry: + + ```console + nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs + ``` + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index 6e97e003e..ebc0e3eff 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -24,6 +24,13 @@ R""( … ``` +* Pin `nixpkgs` in a custom registry to its most recent Git revision: + + ```console + # nix registry pin --registry ./custom-flake-registry.json nixpkgs + ``` + + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md index 4c0eb4947..eecd4c6e7 100644 --- a/src/nix/registry-remove.md +++ b/src/nix/registry-remove.md @@ -8,6 +8,12 @@ R""( # nix registry remove nixpkgs ``` +* Remove the entry `nixpkgs` from a custom registry: + + ```console + # nix registry remove --registry ./custom-flake-registry.json nixpkgs + ``` + # Description This command removes from the user registry any entry for flake diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f9719600f..c496f94f8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -10,6 +10,46 @@ using namespace nix; using namespace nix::flake; + +class RegistryCommand : virtual Args +{ + std::string registry_path; + + std::shared_ptr registry; + +public: + + RegistryCommand() + { + addFlag({ + .longName = "registry", + .description = "The registry to operate on.", + .labels = {"registry"}, + .handler = {®istry_path}, + }); + } + + std::shared_ptr getRegistry() + { + if (registry) return registry; + if (registry_path.empty()) { + registry = fetchers::getUserRegistry(); + } else { + registry = fetchers::getCustomRegistry(registry_path); + } + return registry; + } + + Path getRegistryPath() + { + if (registry_path.empty()) { + return fetchers::getUserRegistryPath(); + } else { + return registry_path; + } + } +}; + struct CmdRegistryList : StoreCommand { std::string description() override @@ -45,7 +85,7 @@ struct CmdRegistryList : StoreCommand } }; -struct CmdRegistryAdd : MixEvalArgs, Command +struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand { std::string fromUrl, toUrl; @@ -71,16 +111,16 @@ struct CmdRegistryAdd : MixEvalArgs, Command { auto fromRef = parseFlakeRef(fromUrl); auto toRef = parseFlakeRef(toUrl); + auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(fromRef.input); - userRegistry->add(fromRef.input, toRef.input, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->remove(fromRef.input); + registry->add(fromRef.input, toRef.input, extraAttrs); + registry->write(getRegistryPath()); } }; -struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command +struct CmdRegistryRemove : RegistryCommand, Command { std::string url; @@ -103,19 +143,21 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command void run() override { - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(parseFlakeRef(url).input); - userRegistry->write(fetchers::getUserRegistryPath()); + auto registry = getRegistry(); + registry->remove(parseFlakeRef(url).input); + registry->write(getRegistryPath()); } }; -struct CmdRegistryPin : virtual Args, EvalCommand +struct CmdRegistryPin : RegistryCommand, EvalCommand { std::string url; + std::string locked; + std::string description() override { - return "pin a flake to its current version in user flake registry"; + return "pin a flake to its current version or to the current version of a flake URL"; } std::string doc() override @@ -128,18 +170,31 @@ struct CmdRegistryPin : virtual Args, EvalCommand CmdRegistryPin() { expectArg("url", &url); + + expectArgs({ + .label = "locked", + .optional = true, + .handler = {&locked}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); } void run(nix::ref store) override { + if (locked.empty()) { + locked = url; + } + auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input.fetch(store); + auto locked_ref = parseFlakeRef(locked); + registry->remove(ref.input); + auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; - userRegistry->add(ref.input, resolved, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->add(ref.input, resolved, extraAttrs); + registry->write(getRegistryPath()); } }; @@ -171,6 +226,7 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { + settings.requireExperimentalFeature(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); command->second->prepare(); diff --git a/src/nix/registry.md b/src/nix/registry.md index 557e5795b..a1674bd2e 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -41,7 +41,7 @@ A registry is a JSON file with the following format: ```json { "version": 2, - [ + "flakes": [ { "from": { "type": "indirect", diff --git a/src/nix/run.cc b/src/nix/run.cc index b5d8ab38a..b01fdebaa 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "run.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -20,45 +21,43 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct RunCommon : virtual Command +namespace nix { + +void runProgramInStore(ref store, + const std::string & program, + const Strings & args) { + stopProgressBar(); - using Command::run; + restoreProcessContext(); - void runProgram(ref store, - const std::string & program, - const Strings & args) - { - stopProgressBar(); + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded program + (which "nix" is), so we exec() a single-threaded helper program + (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast(); - restoreProcessContext(); + if (store2 && store->storeDir != store2->getRealStoreDir()) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + for (auto & arg : args) helperArgs.push_back(arg); - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast(); + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; - for (auto & arg : args) helperArgs.push_back(arg); - - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - - throw SysError("could not execute chroot helper"); - } - - execvp(program.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("unable to execute '%s'", program); + throw SysError("could not execute chroot helper"); } -}; -struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); +} + +} + +struct CmdShell : InstallablesCommand, MixEnvironment { using InstallablesCommand::run; @@ -93,7 +92,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment void run(ref store) override { - auto outPaths = toStorePaths(store, Realise::Outputs, OperateOn::Output, installables); + auto outPaths = toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); @@ -125,13 +124,13 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgram(store, *command.begin(), args); + runProgramInStore(store, *command.begin(), args); } }; static auto rCmdShell = registerCommand("shell"); -struct CmdRun : InstallableCommand, RunCommon +struct CmdRun : InstallableCommand { using InstallableCommand::run; @@ -168,7 +167,7 @@ struct CmdRun : InstallableCommand, RunCommon Strings getDefaultFlakeAttrPathPrefixes() override { - Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + Strings res{"apps." + settings.thisSystem.get() + "."}; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) res.push_back(s); return res; @@ -178,14 +177,12 @@ struct CmdRun : InstallableCommand, RunCommon { auto state = getEvalState(); - auto app = installable->toApp(*state); - - state->store->buildPaths(toDerivedPaths(app.context)); + auto app = installable->toApp(*state).resolve(getEvalStore(), store); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgram(store, app.program, allArgs); + runProgramInStore(store, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh new file mode 100644 index 000000000..6180a87dd --- /dev/null +++ b/src/nix/run.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +void runProgramInStore(ref store, + const std::string & program, + const Strings & args); + +} diff --git a/src/nix/search.cc b/src/nix/search.cc index c52a48d4e..0d8fdd5c2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -62,6 +62,7 @@ struct CmdSearch : InstallableCommand, MixJSON void run(ref store) override { settings.readOnlyMode = true; + evalSettings.enableImportFromDerivation.setDefault(false); // Empty search string should match all packages // Use "^" here instead of ".*" due to differences in resulting highlighting diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 91721219b..29944e748 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -22,10 +22,7 @@ struct CmdShowConfig : Command, MixJSON // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); } else { - std::map settings; - globalConfig.getSettings(settings); - for (auto & s : settings) - logger->cout("%s = %s", s.first, s.second.value); + logger->cout("%s", globalConfig.toKeyValue()); } } }; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2588a011d..c614be68d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -65,18 +65,18 @@ struct CmdShowDerivation : InstallablesCommand auto & outputName = _outputName; // work around clang bug auto outputObj { outputsObj.object(outputName) }; std::visit(overloaded { - [&](DerivationOutputInputAddressed doi) { + [&](const DerivationOutputInputAddressed & doi) { outputObj.attr("path", store->printStorePath(doi.path)); }, - [&](DerivationOutputCAFixed dof) { + [&](const DerivationOutputCAFixed & dof) { outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputCAFloating dof) { + [&](const DerivationOutputCAFloating & dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, - [&](DerivationOutputDeferred) {}, + [&](const DerivationOutputDeferred &) {}, }, output.output); } } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index c64b472b6..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,7 +27,7 @@ struct CmdCopySigs : StorePathsCommand return "copy store path signatures from substituters"; } - void run(ref store, StorePaths storePaths) override + void run(ref store, StorePaths && storePaths) override { if (substituterUris.empty()) throw UsageError("you must specify at least one substituter using '-s'"); @@ -113,7 +113,7 @@ struct CmdSign : StorePathsCommand return "sign store paths"; } - void run(ref store, StorePaths storePaths) override + void run(ref store, StorePaths && storePaths) override { if (secretKeyFile.empty()) throw UsageError("you must specify a secret key file using '-k'"); @@ -218,8 +218,7 @@ struct CmdKey : NixMultiCommand void run() override { if (!command) - throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature("flakes"); + throw UsageError("'nix key' requires a sub-command."); command->second->prepare(); command->second->run(); } diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 10245978e..e4a3cb554 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -30,7 +30,7 @@ struct CmdStoreDelete : StorePathsCommand ; } - void run(ref store, std::vector storePaths) override + void run(ref store, std::vector && storePaths) override { for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index 1c7a4392e..8fcb3639a 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -17,7 +17,7 @@ struct CmdStoreRepair : StorePathsCommand ; } - void run(ref store, std::vector storePaths) override + void run(ref store, std::vector && storePaths) override { for (auto & path : storePaths) store->repairPath(path); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 1721c7f16..e92df1303 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -59,7 +59,7 @@ struct CmdVerify : StorePathsCommand ; } - void run(ref store, StorePaths storePaths) override + void run(ref store, StorePaths && storePaths) override { std::vector> substituters; for (auto & s : substituterUris) @@ -97,15 +97,11 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - std::unique_ptr hashSink; - if (!info->ca) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - store->narFromPath(info->path, *hashSink); + store->narFromPath(info->path, hashSink); - auto hash = hashSink->finish(); + auto hash = hashSink.finish(); if (hash.first != info->narHash) { corrupted++; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 7a4ca5172..2f6b361bb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -62,9 +62,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref store) override { auto package = parseInstallable(store, _package); - auto packagePath = toStorePath(store, Realise::Outputs, operateOn, package); + auto packagePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); auto dependency = parseInstallable(store, _dependency); - auto dependencyPath = toStorePath(store, Realise::Derivation, operateOn, dependency); + auto dependencyPath = toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index c9af0bed3..a70aaf8cb 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,12 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 +| | |__ | | | | | | version 3.9.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . +Copyright (c) 2013-2019 Niels Lohmann . Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,770 +27,44 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ #define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 +#define NLOHMANN_JSON_VERSION_MINOR 9 +#define NLOHMANN_JSON_VERSION_PATCH 1 #include // all_of, find, for_each -#include // assert -#include // and, not, or #include // nullptr_t, ptrdiff_t, size_t #include // hash, less #include // initializer_list #include // istream, ostream #include // random_access_iterator_tag +#include // unique_ptr #include // accumulate #include // string, stoi, to_string #include // declval, forward, move, pair, swap - -// #include -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string #include // vector -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -/*! -@brief default JSONSerializer template argument +// #include -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer> -class basic_json; +#include -/*! -@brief JSON Pointer +// #include -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - -@since version 2.0.0 -*/ -template -class json_pointer; - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} // namespace nlohmann - -#endif - -// #include - - -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them - -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" -#endif - -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) -#else - #define JSON_DEPRECATED -#endif - -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) -#else - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) -#endif - -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER -#endif - -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif - -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif - -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } - -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. - -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> - -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json - -// #include - - -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type - -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; - -template -using uncvref_t = typename std::remove_cv::type>::type; - -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; - -template -struct merge_and_renumber; - -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; - -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; - -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; - -template -using index_sequence_for = make_index_sequence; - -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; - -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; - -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval - -// #include - -// #include - - -#include // random_access_iterator_tag - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; - -template -struct iterator_traits::value>> -{ - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; -}; -} -} - -// #include - -// #include - - -#include - -// #include - - -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ -namespace detail -{ -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - void operator=(nonesuch const&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template